2024-11-11 23:53:16,918 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 23:53:16,931 main DEBUG Took 0.010523 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 23:53:16,931 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 23:53:16,931 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 23:53:16,932 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 23:53:16,934 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,941 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 23:53:16,953 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,955 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,956 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,957 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,958 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,959 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,960 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,961 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,961 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,962 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,963 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,964 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,964 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:53:16,964 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,965 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 23:53:16,966 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:53:16,968 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 23:53:16,970 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 23:53:16,971 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 23:53:16,972 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 23:53:16,973 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 23:53:16,982 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 23:53:16,985 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 23:53:16,987 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 23:53:16,988 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 23:53:16,988 main DEBUG createAppenders(={Console}) 2024-11-11 23:53:16,989 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 23:53:16,989 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 23:53:16,990 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 23:53:16,990 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 23:53:16,991 main DEBUG OutputStream closed 2024-11-11 23:53:16,991 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 23:53:16,991 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 23:53:16,991 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 23:53:17,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 23:53:17,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 23:53:17,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 23:53:17,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 23:53:17,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 23:53:17,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 23:53:17,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 23:53:17,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 23:53:17,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 23:53:17,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 23:53:17,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 23:53:17,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 23:53:17,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 23:53:17,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 23:53:17,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 23:53:17,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 23:53:17,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 23:53:17,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 23:53:17,078 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 23:53:17,078 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 23:53:17,078 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 23:53:17,079 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T23:53:17,344 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c 2024-11-11 23:53:17,347 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 23:53:17,348 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T23:53:17,355 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-11T23:53:17,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T23:53:17,390 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/cluster_bda0d9c8-fb83-479b-2704-c685aeb4fd07, deleteOnExit=true 2024-11-11T23:53:17,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T23:53:17,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/test.cache.data in system properties and HBase conf 2024-11-11T23:53:17,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T23:53:17,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/hadoop.log.dir in system properties and HBase conf 2024-11-11T23:53:17,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T23:53:17,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T23:53:17,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T23:53:17,483 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T23:53:17,573 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T23:53:17,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T23:53:17,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T23:53:17,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T23:53:17,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T23:53:17,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T23:53:17,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T23:53:17,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T23:53:17,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T23:53:17,583 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T23:53:17,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/nfs.dump.dir in system properties and HBase conf 2024-11-11T23:53:17,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/java.io.tmpdir in system properties and HBase conf 2024-11-11T23:53:17,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T23:53:17,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T23:53:17,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T23:53:18,316 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T23:53:18,382 INFO [Time-limited test {}] log.Log(170): Logging initialized @2211ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T23:53:18,442 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:53:18,497 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T23:53:18,515 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T23:53:18,515 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T23:53:18,516 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T23:53:18,526 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:53:18,529 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/hadoop.log.dir/,AVAILABLE} 2024-11-11T23:53:18,530 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T23:53:18,695 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/java.io.tmpdir/jetty-localhost-43153-hadoop-hdfs-3_4_1-tests_jar-_-any-16901416925328222811/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T23:53:18,701 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43153} 2024-11-11T23:53:18,701 INFO [Time-limited test {}] server.Server(415): Started @2530ms 2024-11-11T23:53:19,034 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:53:19,043 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T23:53:19,044 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T23:53:19,044 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T23:53:19,044 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T23:53:19,045 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/hadoop.log.dir/,AVAILABLE} 2024-11-11T23:53:19,046 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T23:53:19,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/java.io.tmpdir/jetty-localhost-38057-hadoop-hdfs-3_4_1-tests_jar-_-any-12387476312389661376/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:53:19,144 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:38057} 2024-11-11T23:53:19,145 INFO [Time-limited test {}] server.Server(415): Started @2974ms 2024-11-11T23:53:19,191 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T23:53:19,621 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/cluster_bda0d9c8-fb83-479b-2704-c685aeb4fd07/data/data1/current/BP-1069310435-172.17.0.2-1731369198105/current, will proceed with Du for space computation calculation, 2024-11-11T23:53:19,621 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/cluster_bda0d9c8-fb83-479b-2704-c685aeb4fd07/data/data2/current/BP-1069310435-172.17.0.2-1731369198105/current, will proceed with Du for space computation calculation, 2024-11-11T23:53:19,657 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T23:53:19,715 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2db5d8e82235c7cc with lease ID 0x84611474746d4691: Processing first storage report for DS-2d108ef8-1524-4288-8a99-aad26cc1ac3b from datanode DatanodeRegistration(127.0.0.1:36259, datanodeUuid=7ec261c1-f9b0-404b-bba9-8b1c60a0fc3a, infoPort=39543, infoSecurePort=0, ipcPort=37723, storageInfo=lv=-57;cid=testClusterID;nsid=684462468;c=1731369198105) 2024-11-11T23:53:19,716 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2db5d8e82235c7cc with lease ID 0x84611474746d4691: from storage DS-2d108ef8-1524-4288-8a99-aad26cc1ac3b node DatanodeRegistration(127.0.0.1:36259, datanodeUuid=7ec261c1-f9b0-404b-bba9-8b1c60a0fc3a, infoPort=39543, infoSecurePort=0, ipcPort=37723, storageInfo=lv=-57;cid=testClusterID;nsid=684462468;c=1731369198105), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T23:53:19,717 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2db5d8e82235c7cc with lease ID 0x84611474746d4691: Processing first storage report for DS-67019b68-a02e-4336-9fd0-e75e1c235627 from datanode DatanodeRegistration(127.0.0.1:36259, datanodeUuid=7ec261c1-f9b0-404b-bba9-8b1c60a0fc3a, infoPort=39543, infoSecurePort=0, ipcPort=37723, storageInfo=lv=-57;cid=testClusterID;nsid=684462468;c=1731369198105) 2024-11-11T23:53:19,717 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2db5d8e82235c7cc with lease ID 0x84611474746d4691: from storage DS-67019b68-a02e-4336-9fd0-e75e1c235627 node DatanodeRegistration(127.0.0.1:36259, datanodeUuid=7ec261c1-f9b0-404b-bba9-8b1c60a0fc3a, infoPort=39543, infoSecurePort=0, ipcPort=37723, storageInfo=lv=-57;cid=testClusterID;nsid=684462468;c=1731369198105), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T23:53:19,734 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c 2024-11-11T23:53:19,810 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/cluster_bda0d9c8-fb83-479b-2704-c685aeb4fd07/zookeeper_0, clientPort=62745, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/cluster_bda0d9c8-fb83-479b-2704-c685aeb4fd07/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/cluster_bda0d9c8-fb83-479b-2704-c685aeb4fd07/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T23:53:19,819 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62745 2024-11-11T23:53:19,828 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:53:19,831 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:53:20,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741825_1001 (size=7) 2024-11-11T23:53:20,440 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 with version=8 2024-11-11T23:53:20,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/hbase-staging 2024-11-11T23:53:20,528 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T23:53:20,741 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b70d29849faf:0 server-side Connection retries=45 2024-11-11T23:53:20,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T23:53:20,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T23:53:20,758 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T23:53:20,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T23:53:20,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T23:53:20,954 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T23:53:21,042 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T23:53:21,068 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T23:53:21,073 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T23:53:21,107 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 42531 (auto-detected) 2024-11-11T23:53:21,108 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T23:53:21,137 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37327 2024-11-11T23:53:21,167 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37327 connecting to ZooKeeper ensemble=127.0.0.1:62745 2024-11-11T23:53:21,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373270x0, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T23:53:21,210 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37327-0x1003134c5200000 connected 2024-11-11T23:53:21,262 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:53:21,266 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:53:21,280 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T23:53:21,286 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075, hbase.cluster.distributed=false 2024-11-11T23:53:21,320 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T23:53:21,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37327 2024-11-11T23:53:21,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37327 2024-11-11T23:53:21,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37327 2024-11-11T23:53:21,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37327 2024-11-11T23:53:21,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37327 2024-11-11T23:53:21,477 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b70d29849faf:0 server-side Connection retries=45 2024-11-11T23:53:21,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T23:53:21,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T23:53:21,480 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T23:53:21,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T23:53:21,481 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T23:53:21,484 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T23:53:21,488 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T23:53:21,490 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42393 2024-11-11T23:53:21,493 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42393 connecting to ZooKeeper ensemble=127.0.0.1:62745 2024-11-11T23:53:21,495 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:53:21,500 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:53:21,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:423930x0, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T23:53:21,519 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:423930x0, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T23:53:21,521 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42393-0x1003134c5200001 connected 2024-11-11T23:53:21,524 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T23:53:21,536 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T23:53:21,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T23:53:21,546 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T23:53:21,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42393 2024-11-11T23:53:21,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42393 2024-11-11T23:53:21,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42393 2024-11-11T23:53:21,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42393 2024-11-11T23:53:21,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42393 2024-11-11T23:53:21,581 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b70d29849faf:37327 2024-11-11T23:53:21,585 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b70d29849faf,37327,1731369200577 2024-11-11T23:53:21,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T23:53:21,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T23:53:21,597 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b70d29849faf,37327,1731369200577 2024-11-11T23:53:21,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:21,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T23:53:21,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:21,625 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T23:53:21,627 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b70d29849faf,37327,1731369200577 from backup master directory 2024-11-11T23:53:21,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b70d29849faf,37327,1731369200577 2024-11-11T23:53:21,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T23:53:21,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T23:53:21,632 WARN [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T23:53:21,632 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b70d29849faf,37327,1731369200577 2024-11-11T23:53:21,634 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T23:53:21,641 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T23:53:21,715 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/hbase.id] with ID: a8859011-a7e7-41c7-b37b-4a7d6cebf423 2024-11-11T23:53:21,715 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/.tmp/hbase.id 2024-11-11T23:53:21,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741826_1002 (size=42) 2024-11-11T23:53:22,136 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/.tmp/hbase.id]:[hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/hbase.id] 2024-11-11T23:53:22,189 INFO [master/b70d29849faf:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:53:22,195 INFO [master/b70d29849faf:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T23:53:22,216 INFO [master/b70d29849faf:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-11T23:53:22,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:22,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:22,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741827_1003 (size=196) 2024-11-11T23:53:22,256 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:53:22,259 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T23:53:22,276 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:22,281 INFO [master/b70d29849faf:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T23:53:22,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741828_1004 (size=1189) 2024-11-11T23:53:22,737 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store 2024-11-11T23:53:22,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741829_1005 (size=34) 2024-11-11T23:53:22,762 INFO [master/b70d29849faf:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T23:53:22,765 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:22,766 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T23:53:22,767 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:53:22,767 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:53:22,768 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T23:53:22,769 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:53:22,769 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:53:22,770 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731369202766Disabling compacts and flushes for region at 1731369202766Disabling writes for close at 1731369202768 (+2 ms)Writing region close event to WAL at 1731369202769 (+1 ms)Closed at 1731369202769 2024-11-11T23:53:22,772 WARN [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/.initializing 2024-11-11T23:53:22,773 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/WALs/b70d29849faf,37327,1731369200577 2024-11-11T23:53:22,783 INFO [master/b70d29849faf:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T23:53:22,800 INFO [master/b70d29849faf:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b70d29849faf%2C37327%2C1731369200577, suffix=, logDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/WALs/b70d29849faf,37327,1731369200577, archiveDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/oldWALs, maxLogs=10 2024-11-11T23:53:22,824 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/WALs/b70d29849faf,37327,1731369200577/b70d29849faf%2C37327%2C1731369200577.1731369202805, exclude list is [], retry=0 2024-11-11T23:53:22,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36259,DS-2d108ef8-1524-4288-8a99-aad26cc1ac3b,DISK] 2024-11-11T23:53:22,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T23:53:22,884 INFO [master/b70d29849faf:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/WALs/b70d29849faf,37327,1731369200577/b70d29849faf%2C37327%2C1731369200577.1731369202805 2024-11-11T23:53:22,885 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39543:39543)] 2024-11-11T23:53:22,885 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:53:22,886 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:22,890 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,892 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,953 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T23:53:22,957 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:22,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:22,960 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T23:53:22,966 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:22,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:22,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T23:53:22,971 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:22,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:22,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T23:53:22,976 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:22,977 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:22,978 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,983 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,985 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,992 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,993 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:22,998 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T23:53:23,003 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:53:23,008 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:53:23,010 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60201736, jitterRate=-0.10292422771453857}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T23:53:23,016 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731369202903Initializing all the Stores at 1731369202905 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369202906 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369202907 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369202907Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369202907Cleaning up temporary data from old regions at 1731369202993 (+86 ms)Region opened successfully at 1731369203016 (+23 ms) 2024-11-11T23:53:23,018 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T23:53:23,054 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ed207d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b70d29849faf/172.17.0.2:0 2024-11-11T23:53:23,089 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T23:53:23,104 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T23:53:23,105 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T23:53:23,108 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T23:53:23,110 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T23:53:23,117 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T23:53:23,117 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T23:53:23,148 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T23:53:23,160 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T23:53:23,162 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T23:53:23,164 INFO [master/b70d29849faf:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T23:53:23,166 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T23:53:23,168 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T23:53:23,170 INFO [master/b70d29849faf:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T23:53:23,174 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T23:53:23,175 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T23:53:23,176 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T23:53:23,178 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T23:53:23,193 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T23:53:23,194 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T23:53:23,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T23:53:23,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T23:53:23,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:23,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:23,200 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b70d29849faf,37327,1731369200577, sessionid=0x1003134c5200000, setting cluster-up flag (Was=false) 2024-11-11T23:53:23,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:23,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:23,214 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T23:53:23,216 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b70d29849faf,37327,1731369200577 2024-11-11T23:53:23,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:23,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:23,224 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T23:53:23,225 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b70d29849faf,37327,1731369200577 2024-11-11T23:53:23,231 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T23:53:23,265 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(746): ClusterId : a8859011-a7e7-41c7-b37b-4a7d6cebf423 2024-11-11T23:53:23,267 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T23:53:23,298 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T23:53:23,307 INFO [master/b70d29849faf:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T23:53:23,312 INFO [master/b70d29849faf:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T23:53:23,318 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b70d29849faf,37327,1731369200577 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T23:53:23,329 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T23:53:23,329 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T23:53:23,330 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b70d29849faf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T23:53:23,330 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b70d29849faf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T23:53:23,330 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b70d29849faf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T23:53:23,330 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b70d29849faf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T23:53:23,330 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b70d29849faf:0, corePoolSize=10, maxPoolSize=10 2024-11-11T23:53:23,330 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,331 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b70d29849faf:0, corePoolSize=2, maxPoolSize=2 2024-11-11T23:53:23,331 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,332 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731369233332 2024-11-11T23:53:23,333 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T23:53:23,333 DEBUG [RS:0;b70d29849faf:42393 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3f04f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b70d29849faf/172.17.0.2:0 2024-11-11T23:53:23,334 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T23:53:23,335 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T23:53:23,337 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T23:53:23,337 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T23:53:23,340 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T23:53:23,340 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T23:53:23,340 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T23:53:23,340 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T23:53:23,341 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,344 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:23,344 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T23:53:23,344 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T23:53:23,346 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T23:53:23,346 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T23:53:23,348 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T23:53:23,349 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T23:53:23,351 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b70d29849faf:42393 2024-11-11T23:53:23,355 INFO [RS:0;b70d29849faf:42393 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T23:53:23,355 INFO [RS:0;b70d29849faf:42393 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T23:53:23,355 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T23:53:23,356 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.large.0-1731369203350,5,FailOnTimeoutGroup] 2024-11-11T23:53:23,357 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.small.0-1731369203357,5,FailOnTimeoutGroup] 2024-11-11T23:53:23,358 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,358 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T23:53:23,358 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(2659): reportForDuty to master=b70d29849faf,37327,1731369200577 with port=42393, startcode=1731369201430 2024-11-11T23:53:23,359 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,360 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741831_1007 (size=1321) 2024-11-11T23:53:23,374 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T23:53:23,375 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 2024-11-11T23:53:23,376 DEBUG [RS:0;b70d29849faf:42393 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T23:53:23,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741832_1008 (size=32) 2024-11-11T23:53:23,417 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50813, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T23:53:23,425 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b70d29849faf,42393,1731369201430 2024-11-11T23:53:23,428 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37327 {}] master.ServerManager(517): Registering regionserver=b70d29849faf,42393,1731369201430 2024-11-11T23:53:23,444 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 2024-11-11T23:53:23,444 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46597 2024-11-11T23:53:23,444 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T23:53:23,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T23:53:23,449 DEBUG [RS:0;b70d29849faf:42393 {}] zookeeper.ZKUtil(111): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b70d29849faf,42393,1731369201430 2024-11-11T23:53:23,450 WARN [RS:0;b70d29849faf:42393 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T23:53:23,450 INFO [RS:0;b70d29849faf:42393 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T23:53:23,450 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/WALs/b70d29849faf,42393,1731369201430 2024-11-11T23:53:23,452 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b70d29849faf,42393,1731369201430] 2024-11-11T23:53:23,478 INFO [RS:0;b70d29849faf:42393 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T23:53:23,497 INFO [RS:0;b70d29849faf:42393 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T23:53:23,502 INFO [RS:0;b70d29849faf:42393 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T23:53:23,502 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,503 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T23:53:23,509 INFO [RS:0;b70d29849faf:42393 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T23:53:23,511 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,511 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,511 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,512 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,512 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,512 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,512 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b70d29849faf:0, corePoolSize=2, maxPoolSize=2 2024-11-11T23:53:23,512 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,513 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,513 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,513 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,513 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,513 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:53:23,513 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b70d29849faf:0, corePoolSize=3, maxPoolSize=3 2024-11-11T23:53:23,514 DEBUG [RS:0;b70d29849faf:42393 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0, corePoolSize=3, maxPoolSize=3 2024-11-11T23:53:23,515 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,515 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,516 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,516 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,516 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,516 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,42393,1731369201430-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T23:53:23,542 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T23:53:23,544 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,42393,1731369201430-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,544 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,544 INFO [RS:0;b70d29849faf:42393 {}] regionserver.Replication(171): b70d29849faf,42393,1731369201430 started 2024-11-11T23:53:23,565 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:23,565 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1482): Serving as b70d29849faf,42393,1731369201430, RpcServer on b70d29849faf/172.17.0.2:42393, sessionid=0x1003134c5200001 2024-11-11T23:53:23,566 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T23:53:23,566 DEBUG [RS:0;b70d29849faf:42393 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b70d29849faf,42393,1731369201430 2024-11-11T23:53:23,566 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b70d29849faf,42393,1731369201430' 2024-11-11T23:53:23,566 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T23:53:23,567 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T23:53:23,568 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T23:53:23,568 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T23:53:23,568 DEBUG [RS:0;b70d29849faf:42393 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b70d29849faf,42393,1731369201430 2024-11-11T23:53:23,568 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b70d29849faf,42393,1731369201430' 2024-11-11T23:53:23,568 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T23:53:23,569 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T23:53:23,570 DEBUG [RS:0;b70d29849faf:42393 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T23:53:23,570 INFO [RS:0;b70d29849faf:42393 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T23:53:23,570 INFO [RS:0;b70d29849faf:42393 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T23:53:23,675 INFO [RS:0;b70d29849faf:42393 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T23:53:23,678 INFO [RS:0;b70d29849faf:42393 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b70d29849faf%2C42393%2C1731369201430, suffix=, logDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/WALs/b70d29849faf,42393,1731369201430, archiveDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/oldWALs, maxLogs=32 2024-11-11T23:53:23,695 DEBUG [RS:0;b70d29849faf:42393 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/WALs/b70d29849faf,42393,1731369201430/b70d29849faf%2C42393%2C1731369201430.1731369203681, exclude list is [], retry=0 2024-11-11T23:53:23,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36259,DS-2d108ef8-1524-4288-8a99-aad26cc1ac3b,DISK] 2024-11-11T23:53:23,704 INFO [RS:0;b70d29849faf:42393 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/WALs/b70d29849faf,42393,1731369201430/b70d29849faf%2C42393%2C1731369201430.1731369203681 2024-11-11T23:53:23,705 DEBUG [RS:0;b70d29849faf:42393 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39543:39543)] 2024-11-11T23:53:23,790 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:23,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T23:53:23,796 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T23:53:23,796 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:23,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:23,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T23:53:23,800 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T23:53:23,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:23,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:23,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T23:53:23,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T23:53:23,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:23,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:23,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T23:53:23,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T23:53:23,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:23,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:23,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T23:53:23,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740 2024-11-11T23:53:23,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740 2024-11-11T23:53:23,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T23:53:23,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T23:53:23,825 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:53:23,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T23:53:23,837 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:53:23,839 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61296255, jitterRate=-0.08661462366580963}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:53:23,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731369203790Initializing all the Stores at 1731369203792 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369203792Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369203792Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369203792Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369203792Cleaning up temporary data from old regions at 1731369203824 (+32 ms)Region opened successfully at 1731369203842 (+18 ms) 2024-11-11T23:53:23,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T23:53:23,842 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T23:53:23,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T23:53:23,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T23:53:23,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T23:53:23,846 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T23:53:23,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731369203842Disabling compacts and flushes for region at 1731369203842Disabling writes for close at 1731369203843 (+1 ms)Writing region close event to WAL at 1731369203845 (+2 ms)Closed at 1731369203846 (+1 ms) 2024-11-11T23:53:23,851 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T23:53:23,852 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T23:53:23,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T23:53:23,871 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T23:53:23,874 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T23:53:24,027 DEBUG [b70d29849faf:37327 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T23:53:24,041 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:24,049 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b70d29849faf,42393,1731369201430, state=OPENING 2024-11-11T23:53:24,055 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T23:53:24,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:24,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:53:24,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T23:53:24,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T23:53:24,059 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T23:53:24,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:53:24,244 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T23:53:24,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41125, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T23:53:24,264 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T23:53:24,264 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T23:53:24,265 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T23:53:24,269 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b70d29849faf%2C42393%2C1731369201430.meta, suffix=.meta, logDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/WALs/b70d29849faf,42393,1731369201430, archiveDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/oldWALs, maxLogs=32 2024-11-11T23:53:24,284 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/WALs/b70d29849faf,42393,1731369201430/b70d29849faf%2C42393%2C1731369201430.meta.1731369204271.meta, exclude list is [], retry=0 2024-11-11T23:53:24,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36259,DS-2d108ef8-1524-4288-8a99-aad26cc1ac3b,DISK] 2024-11-11T23:53:24,292 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/WALs/b70d29849faf,42393,1731369201430/b70d29849faf%2C42393%2C1731369201430.meta.1731369204271.meta 2024-11-11T23:53:24,296 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39543:39543)] 2024-11-11T23:53:24,296 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:53:24,298 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T23:53:24,300 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T23:53:24,305 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T23:53:24,309 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T23:53:24,310 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:24,310 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T23:53:24,310 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T23:53:24,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T23:53:24,318 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T23:53:24,318 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:24,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:24,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T23:53:24,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T23:53:24,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:24,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:24,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T23:53:24,326 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T23:53:24,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:24,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:24,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T23:53:24,331 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T23:53:24,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:24,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:53:24,333 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T23:53:24,335 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740 2024-11-11T23:53:24,338 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740 2024-11-11T23:53:24,341 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T23:53:24,341 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T23:53:24,343 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:53:24,346 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T23:53:24,349 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59447384, jitterRate=-0.11416494846343994}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:53:24,349 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T23:53:24,351 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731369204311Writing region info on filesystem at 1731369204311Initializing all the Stores at 1731369204314 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369204314Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369204315 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369204315Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369204315Cleaning up temporary data from old regions at 1731369204341 (+26 ms)Running coprocessor post-open hooks at 1731369204349 (+8 ms)Region opened successfully at 1731369204351 (+2 ms) 2024-11-11T23:53:24,359 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731369204232 2024-11-11T23:53:24,374 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T23:53:24,375 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T23:53:24,376 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:24,380 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b70d29849faf,42393,1731369201430, state=OPEN 2024-11-11T23:53:24,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T23:53:24,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T23:53:24,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T23:53:24,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T23:53:24,384 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:24,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T23:53:24,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b70d29849faf,42393,1731369201430 in 323 msec 2024-11-11T23:53:24,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T23:53:24,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 533 msec 2024-11-11T23:53:24,403 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T23:53:24,403 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T23:53:24,427 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:24,429 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:24,454 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:24,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54263, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:24,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2210 sec 2024-11-11T23:53:24,482 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731369204482, completionTime=-1 2024-11-11T23:53:24,485 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T23:53:24,485 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T23:53:24,515 INFO [master/b70d29849faf:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T23:53:24,515 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731369264515 2024-11-11T23:53:24,516 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731369324516 2024-11-11T23:53:24,516 INFO [master/b70d29849faf:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-11T23:53:24,518 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,37327,1731369200577-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:24,518 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,37327,1731369200577-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:24,519 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,37327,1731369200577-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:24,520 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b70d29849faf:37327, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:24,521 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:24,521 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:24,528 DEBUG [master/b70d29849faf:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T23:53:24,553 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.921sec 2024-11-11T23:53:24,555 INFO [master/b70d29849faf:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T23:53:24,556 INFO [master/b70d29849faf:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T23:53:24,557 INFO [master/b70d29849faf:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T23:53:24,558 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T23:53:24,558 INFO [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T23:53:24,559 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,37327,1731369200577-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T23:53:24,560 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,37327,1731369200577-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T23:53:24,570 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T23:53:24,572 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T23:53:24,572 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,37327,1731369200577-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:53:24,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:24,598 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T23:53:24,599 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T23:53:24,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:24,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:24,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17703dee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:24,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:24,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:24,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:24,641 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:24,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:24,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b70d29849faf,37327,1731369200577 2024-11-11T23:53:24,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=564, ProcessCount=11, AvailableMemoryMB=5402 2024-11-11T23:53:24,716 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T23:53:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b70d29849faf,37327,1731369200577 2024-11-11T23:53:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55eec305 2024-11-11T23:53:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T23:53:24,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T23:53:24,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:53:24,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:53:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:24,754 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:53:24,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T23:53:24,757 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:24,761 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:53:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T23:53:24,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741835_1011 (size=1061) 2024-11-11T23:53:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T23:53:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T23:53:25,178 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 2024-11-11T23:53:25,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741836_1012 (size=53) 2024-11-11T23:53:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T23:53:25,589 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:25,590 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6b2061b8554b1e7422f6c037ae02cccd, disabling compactions & flushes 2024-11-11T23:53:25,590 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:25,590 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:25,590 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. after waiting 0 ms 2024-11-11T23:53:25,590 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:25,590 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:25,590 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6b2061b8554b1e7422f6c037ae02cccd: Waiting for close lock at 1731369205590Disabling compacts and flushes for region at 1731369205590Disabling writes for close at 1731369205590Writing region close event to WAL at 1731369205590Closed at 1731369205590 2024-11-11T23:53:25,593 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:53:25,598 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731369205593"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369205593"}]},"ts":"1731369205593"} 2024-11-11T23:53:25,607 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:53:25,614 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:53:25,618 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369205614"}]},"ts":"1731369205614"} 2024-11-11T23:53:25,624 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T23:53:25,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, ASSIGN}] 2024-11-11T23:53:25,632 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, ASSIGN 2024-11-11T23:53:25,635 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, ASSIGN; state=OFFLINE, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=false 2024-11-11T23:53:25,788 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6b2061b8554b1e7422f6c037ae02cccd, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:25,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, ASSIGN because future has completed 2024-11-11T23:53:25,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:53:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T23:53:25,968 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:25,968 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:53:25,969 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,969 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:25,969 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,969 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,972 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,976 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:25,976 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b2061b8554b1e7422f6c037ae02cccd columnFamilyName A 2024-11-11T23:53:25,976 DEBUG [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:25,978 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.HStore(327): Store=6b2061b8554b1e7422f6c037ae02cccd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:25,978 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,981 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:25,981 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b2061b8554b1e7422f6c037ae02cccd columnFamilyName B 2024-11-11T23:53:25,981 DEBUG [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:25,982 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.HStore(327): Store=6b2061b8554b1e7422f6c037ae02cccd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:25,982 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,985 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:25,985 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b2061b8554b1e7422f6c037ae02cccd columnFamilyName C 2024-11-11T23:53:25,986 DEBUG [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:25,987 INFO [StoreOpener-6b2061b8554b1e7422f6c037ae02cccd-1 {}] regionserver.HStore(327): Store=6b2061b8554b1e7422f6c037ae02cccd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:25,987 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:25,988 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,990 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,991 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,994 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,994 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:25,995 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:53:25,998 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:26,005 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:53:26,007 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6b2061b8554b1e7422f6c037ae02cccd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62315580, jitterRate=-0.07142549753189087}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:53:26,007 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:26,008 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6b2061b8554b1e7422f6c037ae02cccd: Running coprocessor pre-open hook at 1731369205969Writing region info on filesystem at 1731369205969Initializing all the Stores at 1731369205971 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369205971Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369205972 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369205972Cleaning up temporary data from old regions at 1731369205994 (+22 ms)Running coprocessor post-open hooks at 1731369206007 (+13 ms)Region opened successfully at 1731369206008 (+1 ms) 2024-11-11T23:53:26,012 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., pid=6, masterSystemTime=1731369205955 2024-11-11T23:53:26,016 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:26,016 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:26,017 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6b2061b8554b1e7422f6c037ae02cccd, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:26,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:53:26,024 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37327 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b70d29849faf,42393,1731369201430, table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T23:53:26,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T23:53:26,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 in 227 msec 2024-11-11T23:53:26,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T23:53:26,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, ASSIGN in 403 msec 2024-11-11T23:53:26,038 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:53:26,039 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369206038"}]},"ts":"1731369206038"} 2024-11-11T23:53:26,043 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T23:53:26,046 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:53:26,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3060 sec 2024-11-11T23:53:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T23:53:26,926 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:26,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:26,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:26,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2315f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:26,936 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35eddcd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,939 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:26,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:26,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T23:53:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b70d29849faf,37327,1731369200577 2024-11-11T23:53:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@19bc5c32 2024-11-11T23:53:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T23:53:26,953 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T23:53:26,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:26,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:26,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7290076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:26,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9c1aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:26,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:26,970 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:26,976 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:26,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:26,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef8cf77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:26,990 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:26,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:26,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:26,996 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebcbe76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:27,005 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d0905d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:27,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:27,011 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:27,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:27,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bfbcabb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:27,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:27,014 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876c349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:27,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:27,020 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:27,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40135d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:27,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:27,024 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f58bfae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:27,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:27,031 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35351cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:27,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516bf0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:27,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:27,039 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9a1dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:27,042 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:27,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab586a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:27,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:27,046 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b6e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:27,048 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4c23c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:27,059 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,063 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,065 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,065 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,067 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,067 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,069 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,070 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,087 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:27,089 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:27,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T23:53:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:53:27,109 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:53:27,126 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:27,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:27,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:53:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:53:27,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:27,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:27,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:27,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:27,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:27,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:27,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:27,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/a38e230ad065473582b53f8677ac01df is 50, key is test_row_0/A:col10/1731369207186/Put/seqid=0 2024-11-11T23:53:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741837_1013 (size=12001) 2024-11-11T23:53:27,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/a38e230ad065473582b53f8677ac01df 2024-11-11T23:53:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:53:27,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369267475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:27,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369267489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:27,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:27,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731369207328, max timestamp=1731369207487Num uniques 859; 2024-11-11T23:53:27,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T23:53:27,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369207328, max timestamp=1731369207492Num uniques 859; 2024-11-11T23:53:27,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:27,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369207328, max timestamp=1731369207490Num uniques 859; 2024-11-11T23:53:27,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:27,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:27,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:27,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:27,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/b8ee74b0187c493ca8e213e44d261bb4 is 50, key is test_row_0/B:col10/1731369207186/Put/seqid=0 2024-11-11T23:53:27,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741838_1014 (size=12001) 2024-11-11T23:53:27,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/b8ee74b0187c493ca8e213e44d261bb4 2024-11-11T23:53:27,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369267643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369267647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:27,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/a77175e401b04b17bee91f6546d4fb80 is 50, key is test_row_0/C:col10/1731369207186/Put/seqid=0 2024-11-11T23:53:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369267652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:27,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369267649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:27,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369267655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:27,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741839_1015 (size=12001) 2024-11-11T23:53:27,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/a77175e401b04b17bee91f6546d4fb80 2024-11-11T23:53:27,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:27,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:27,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:27,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:27,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/a38e230ad065473582b53f8677ac01df as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a38e230ad065473582b53f8677ac01df 2024-11-11T23:53:27,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a38e230ad065473582b53f8677ac01df, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T23:53:27,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/b8ee74b0187c493ca8e213e44d261bb4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b8ee74b0187c493ca8e213e44d261bb4 2024-11-11T23:53:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:53:27,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b8ee74b0187c493ca8e213e44d261bb4, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T23:53:27,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/a77175e401b04b17bee91f6546d4fb80 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/a77175e401b04b17bee91f6546d4fb80 2024-11-11T23:53:27,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369267756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:27,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/a77175e401b04b17bee91f6546d4fb80, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T23:53:27,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.78 KB/136995 for 6b2061b8554b1e7422f6c037ae02cccd in 587ms, sequenceid=12, compaction requested=false 2024-11-11T23:53:27,787 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T23:53:27,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:27,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369207619, max timestamp=1731369207793Num uniques 1018; 2024-11-11T23:53:27,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:53:27,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369207619, max timestamp=1731369207804Num uniques 1009; 2024-11-11T23:53:27,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-11T23:53:27,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=71.56 KB heapSize=198.72 KB 2024-11-11T23:53:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:27,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:53:27,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/efd4a0ee650e49ba8b8e58d4db18d5ac is 50, key is test_row_0/A:col10/1731369207630/Put/seqid=0 2024-11-11T23:53:27,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:27,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:27,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:27,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:27,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:27,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741840_1016 (size=12001) 2024-11-11T23:53:27,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/efd4a0ee650e49ba8b8e58d4db18d5ac 2024-11-11T23:53:27,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/c4307fe9a3f74f979ef2fbc2acdbbdf7 is 50, key is test_row_0/B:col10/1731369207630/Put/seqid=0 2024-11-11T23:53:27,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741841_1017 (size=12001) 2024-11-11T23:53:27,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/c4307fe9a3f74f979ef2fbc2acdbbdf7 2024-11-11T23:53:27,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/1243c91eb74841fb89f25a8829d51fcb is 50, key is test_row_0/C:col10/1731369207630/Put/seqid=0 2024-11-11T23:53:27,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369267955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369267976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369267988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369267998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369267993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741842_1018 (size=12001) 2024-11-11T23:53:28,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/1243c91eb74841fb89f25a8829d51fcb 2024-11-11T23:53:28,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/efd4a0ee650e49ba8b8e58d4db18d5ac as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/efd4a0ee650e49ba8b8e58d4db18d5ac 2024-11-11T23:53:28,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/efd4a0ee650e49ba8b8e58d4db18d5ac, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T23:53:28,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/c4307fe9a3f74f979ef2fbc2acdbbdf7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c4307fe9a3f74f979ef2fbc2acdbbdf7 2024-11-11T23:53:28,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c4307fe9a3f74f979ef2fbc2acdbbdf7, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T23:53:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/1243c91eb74841fb89f25a8829d51fcb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/1243c91eb74841fb89f25a8829d51fcb 2024-11-11T23:53:28,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/1243c91eb74841fb89f25a8829d51fcb, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T23:53:28,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=107.34 KB/109920 for 6b2061b8554b1e7422f6c037ae02cccd in 288ms, sequenceid=51, compaction requested=false 2024-11-11T23:53:28,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:28,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T23:53:28,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:28,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:28,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:28,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:28,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:28,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:28,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/89d6662bc63441e1bb5bc153ba6921f0 is 50, key is test_row_0/A:col10/1731369207950/Put/seqid=0 2024-11-11T23:53:28,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369268162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741843_1019 (size=14341) 2024-11-11T23:53:28,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/89d6662bc63441e1bb5bc153ba6921f0 2024-11-11T23:53:28,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369268172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369268175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369268181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/98ac11d93ec740c7a725509ee22ef1db is 50, key is test_row_0/B:col10/1731369207950/Put/seqid=0 2024-11-11T23:53:28,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741844_1020 (size=12001) 2024-11-11T23:53:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:53:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369268285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369268295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369268295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369268305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369268315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:28,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:28,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369268505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369268516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369268516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369268525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:28,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/98ac11d93ec740c7a725509ee22ef1db 2024-11-11T23:53:28,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/31d51b3aabc242989f0b7a0164f66c66 is 50, key is test_row_0/C:col10/1731369207950/Put/seqid=0 2024-11-11T23:53:28,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741845_1021 (size=12001) 2024-11-11T23:53:28,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/31d51b3aabc242989f0b7a0164f66c66 2024-11-11T23:53:28,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/89d6662bc63441e1bb5bc153ba6921f0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/89d6662bc63441e1bb5bc153ba6921f0 2024-11-11T23:53:28,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/89d6662bc63441e1bb5bc153ba6921f0, entries=200, sequenceid=71, filesize=14.0 K 2024-11-11T23:53:28,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/98ac11d93ec740c7a725509ee22ef1db as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/98ac11d93ec740c7a725509ee22ef1db 2024-11-11T23:53:28,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/98ac11d93ec740c7a725509ee22ef1db, entries=150, sequenceid=71, filesize=11.7 K 2024-11-11T23:53:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/31d51b3aabc242989f0b7a0164f66c66 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/31d51b3aabc242989f0b7a0164f66c66 2024-11-11T23:53:28,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/31d51b3aabc242989f0b7a0164f66c66, entries=150, sequenceid=71, filesize=11.7 K 2024-11-11T23:53:28,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6b2061b8554b1e7422f6c037ae02cccd in 669ms, sequenceid=71, compaction requested=true 2024-11-11T23:53:28,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:28,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T23:53:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,808 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T23:53:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:28,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:28,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:28,821 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:28,821 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:28,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:28,828 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:28,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:28,831 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:28,832 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:28,832 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/bab34a8d1c5a4be7a582b129f197c5c5 is 50, key is test_row_0/A:col10/1731369208171/Put/seqid=0 2024-11-11T23:53:28,833 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a38e230ad065473582b53f8677ac01df, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/efd4a0ee650e49ba8b8e58d4db18d5ac, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/89d6662bc63441e1bb5bc153ba6921f0] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=37.4 K 2024-11-11T23:53:28,832 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,833 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b8ee74b0187c493ca8e213e44d261bb4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c4307fe9a3f74f979ef2fbc2acdbbdf7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/98ac11d93ec740c7a725509ee22ef1db] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.2 K 2024-11-11T23:53:28,835 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38e230ad065473582b53f8677ac01df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369207119 2024-11-11T23:53:28,837 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd4a0ee650e49ba8b8e58d4db18d5ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731369207619 2024-11-11T23:53:28,838 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ee74b0187c493ca8e213e44d261bb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369207119 2024-11-11T23:53:28,839 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d6662bc63441e1bb5bc153ba6921f0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731369207948 2024-11-11T23:53:28,841 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c4307fe9a3f74f979ef2fbc2acdbbdf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731369207619 2024-11-11T23:53:28,842 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ac11d93ec740c7a725509ee22ef1db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731369207950 2024-11-11T23:53:28,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741846_1022 (size=12001) 2024-11-11T23:53:28,858 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/bab34a8d1c5a4be7a582b129f197c5c5 2024-11-11T23:53:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/f9ebbaea87074ed4a5e9bab74c4b512b is 50, key is test_row_0/B:col10/1731369208171/Put/seqid=0 2024-11-11T23:53:28,909 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#10 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:28,909 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:28,910 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/8b3f84b5ca174282808eea81f337a332 is 50, key is test_row_0/B:col10/1731369207950/Put/seqid=0 2024-11-11T23:53:28,910 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7f79bba16c804ebe8b34177589be14ae is 50, key is test_row_0/A:col10/1731369207950/Put/seqid=0 2024-11-11T23:53:28,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741847_1023 (size=12001) 2024-11-11T23:53:28,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741848_1024 (size=12104) 2024-11-11T23:53:28,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741849_1025 (size=12104) 2024-11-11T23:53:28,943 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7f79bba16c804ebe8b34177589be14ae as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7f79bba16c804ebe8b34177589be14ae 2024-11-11T23:53:28,946 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/8b3f84b5ca174282808eea81f337a332 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8b3f84b5ca174282808eea81f337a332 2024-11-11T23:53:28,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369268885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:28,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731369208854, max timestamp=1731369208924Num uniques 859; 2024-11-11T23:53:28,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:53:28,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369208875, max timestamp=1731369208924Num uniques 859; 2024-11-11T23:53:28,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:28,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369208875, max timestamp=1731369208924Num uniques 859; 2024-11-11T23:53:28,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:28,979 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into 8b3f84b5ca174282808eea81f337a332(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:28,979 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into 7f79bba16c804ebe8b34177589be14ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:28,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:28,981 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:28,984 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369208817; duration=0sec 2024-11-11T23:53:28,984 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:28,985 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:28,985 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:28,985 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369208802; duration=0sec 2024-11-11T23:53:28,986 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:28,986 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:28,989 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:28,989 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:28,989 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:28,989 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/a77175e401b04b17bee91f6546d4fb80, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/1243c91eb74841fb89f25a8829d51fcb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/31d51b3aabc242989f0b7a0164f66c66] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.2 K 2024-11-11T23:53:28,990 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a77175e401b04b17bee91f6546d4fb80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369207119 2024-11-11T23:53:28,993 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1243c91eb74841fb89f25a8829d51fcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731369207619 2024-11-11T23:53:28,994 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d51b3aabc242989f0b7a0164f66c66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731369207950 2024-11-11T23:53:29,027 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:29,028 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/003cb90556c144c2b0f6a28e6715d11e is 50, key is test_row_0/C:col10/1731369207950/Put/seqid=0 2024-11-11T23:53:29,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369269028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369269028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369269031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369269031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741850_1026 (size=12104) 2024-11-11T23:53:29,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369269065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,076 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/003cb90556c144c2b0f6a28e6715d11e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/003cb90556c144c2b0f6a28e6715d11e 2024-11-11T23:53:29,101 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into 003cb90556c144c2b0f6a28e6715d11e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:29,101 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:29,101 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369208817; duration=0sec 2024-11-11T23:53:29,101 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:29,102 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:29,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369269146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369269146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369269146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369269146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:53:29,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369269285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,321 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/f9ebbaea87074ed4a5e9bab74c4b512b 2024-11-11T23:53:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/ac3010eacd4f4fc8bec75765593a88c9 is 50, key is test_row_0/C:col10/1731369208171/Put/seqid=0 2024-11-11T23:53:29,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369269357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369269366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369269366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369269366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741851_1027 (size=12001) 2024-11-11T23:53:29,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369269594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369269675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369269676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369269676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369269676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:29,781 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/ac3010eacd4f4fc8bec75765593a88c9 2024-11-11T23:53:29,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/bab34a8d1c5a4be7a582b129f197c5c5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/bab34a8d1c5a4be7a582b129f197c5c5 2024-11-11T23:53:29,808 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/bab34a8d1c5a4be7a582b129f197c5c5, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T23:53:29,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/f9ebbaea87074ed4a5e9bab74c4b512b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f9ebbaea87074ed4a5e9bab74c4b512b 2024-11-11T23:53:29,822 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:53:29,823 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f9ebbaea87074ed4a5e9bab74c4b512b, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T23:53:29,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/ac3010eacd4f4fc8bec75765593a88c9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/ac3010eacd4f4fc8bec75765593a88c9 2024-11-11T23:53:29,839 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/ac3010eacd4f4fc8bec75765593a88c9, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T23:53:29,842 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=129.71 KB/132820 for 6b2061b8554b1e7422f6c037ae02cccd in 1034ms, sequenceid=83, compaction requested=false 2024-11-11T23:53:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T23:53:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T23:53:29,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T23:53:29,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7150 sec 2024-11-11T23:53:29,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.7600 sec 2024-11-11T23:53:29,912 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T23:53:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:30,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-11T23:53:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:30,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:30,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:30,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:30,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/69e449cdc4db4564ba03afa1023aa0e3 is 50, key is test_row_0/A:col10/1731369210105/Put/seqid=0 2024-11-11T23:53:30,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741852_1028 (size=14341) 2024-11-11T23:53:30,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/69e449cdc4db4564ba03afa1023aa0e3 2024-11-11T23:53:30,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369270162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:30,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/37e8e410aeee4ca0b4e82b0b99645798 is 50, key is test_row_0/B:col10/1731369210105/Put/seqid=0 2024-11-11T23:53:30,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369270195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:30,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369270196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369270196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:30,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369270196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:30,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741853_1029 (size=12001) 2024-11-11T23:53:30,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/37e8e410aeee4ca0b4e82b0b99645798 2024-11-11T23:53:30,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/9a553b6adb4e4171adcfe447a42ba22a is 50, key is test_row_0/C:col10/1731369210105/Put/seqid=0 2024-11-11T23:53:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741854_1030 (size=12001) 2024-11-11T23:53:30,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/9a553b6adb4e4171adcfe447a42ba22a 2024-11-11T23:53:30,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/69e449cdc4db4564ba03afa1023aa0e3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/69e449cdc4db4564ba03afa1023aa0e3 2024-11-11T23:53:30,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369270275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:30,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/69e449cdc4db4564ba03afa1023aa0e3, entries=200, sequenceid=122, filesize=14.0 K 2024-11-11T23:53:30,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/37e8e410aeee4ca0b4e82b0b99645798 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/37e8e410aeee4ca0b4e82b0b99645798 2024-11-11T23:53:30,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/37e8e410aeee4ca0b4e82b0b99645798, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T23:53:30,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/9a553b6adb4e4171adcfe447a42ba22a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/9a553b6adb4e4171adcfe447a42ba22a 2024-11-11T23:53:30,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/9a553b6adb4e4171adcfe447a42ba22a, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T23:53:30,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for 6b2061b8554b1e7422f6c037ae02cccd in 206ms, sequenceid=122, compaction requested=true 2024-11-11T23:53:30,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:30,315 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:30,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:30,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:30,316 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:30,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:30,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:30,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:30,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:30,317 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:30,318 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:30,318 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:30,318 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7f79bba16c804ebe8b34177589be14ae, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/bab34a8d1c5a4be7a582b129f197c5c5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/69e449cdc4db4564ba03afa1023aa0e3] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=37.5 K 2024-11-11T23:53:30,318 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:30,319 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:30,319 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:30,319 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8b3f84b5ca174282808eea81f337a332, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f9ebbaea87074ed4a5e9bab74c4b512b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/37e8e410aeee4ca0b4e82b0b99645798] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.3 K 2024-11-11T23:53:30,319 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f79bba16c804ebe8b34177589be14ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731369207950 2024-11-11T23:53:30,320 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b3f84b5ca174282808eea81f337a332, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731369207950 2024-11-11T23:53:30,320 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting bab34a8d1c5a4be7a582b129f197c5c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731369208168 2024-11-11T23:53:30,321 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9ebbaea87074ed4a5e9bab74c4b512b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731369208168 2024-11-11T23:53:30,323 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e8e410aeee4ca0b4e82b0b99645798, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731369209021 2024-11-11T23:53:30,323 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 69e449cdc4db4564ba03afa1023aa0e3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731369209021 2024-11-11T23:53:30,358 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:30,359 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/2107f983dc4b46b390458390f64ca911 is 50, key is test_row_0/B:col10/1731369210105/Put/seqid=0 2024-11-11T23:53:30,364 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:30,364 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/573d948c57124e3cb52511d955ab6e90 is 50, key is test_row_0/A:col10/1731369210105/Put/seqid=0 2024-11-11T23:53:30,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741855_1031 (size=12207) 2024-11-11T23:53:30,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741856_1032 (size=12207) 2024-11-11T23:53:30,402 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/573d948c57124e3cb52511d955ab6e90 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/573d948c57124e3cb52511d955ab6e90 2024-11-11T23:53:30,419 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into 573d948c57124e3cb52511d955ab6e90(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:30,419 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:30,420 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369210315; duration=0sec 2024-11-11T23:53:30,420 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:30,420 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:30,420 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:30,423 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:30,423 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:30,423 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:30,424 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/003cb90556c144c2b0f6a28e6715d11e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/ac3010eacd4f4fc8bec75765593a88c9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/9a553b6adb4e4171adcfe447a42ba22a] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.3 K 2024-11-11T23:53:30,427 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 003cb90556c144c2b0f6a28e6715d11e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731369207950 2024-11-11T23:53:30,428 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3010eacd4f4fc8bec75765593a88c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731369208168 2024-11-11T23:53:30,429 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a553b6adb4e4171adcfe447a42ba22a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731369209021 2024-11-11T23:53:30,468 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:30,469 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/f3344eb4a9f34fbb9188aed49aa0aec9 is 50, key is test_row_0/C:col10/1731369210105/Put/seqid=0 2024-11-11T23:53:30,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741857_1033 (size=12207) 2024-11-11T23:53:30,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T23:53:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:30,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:30,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:30,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:30,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:30,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:30,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:30,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/cc2be082a02b4477a7614ac49b42886d is 50, key is test_row_0/A:col10/1731369210489/Put/seqid=0 2024-11-11T23:53:30,508 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/f3344eb4a9f34fbb9188aed49aa0aec9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f3344eb4a9f34fbb9188aed49aa0aec9 2024-11-11T23:53:30,529 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into f3344eb4a9f34fbb9188aed49aa0aec9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:30,529 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:30,529 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369210316; duration=0sec 2024-11-11T23:53:30,531 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:30,531 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:30,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741858_1034 (size=14441) 2024-11-11T23:53:30,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369210585, max timestamp=1731369210623Num uniques 859; 2024-11-11T23:53:30,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369210585, max timestamp=1731369210623Num uniques 859; 2024-11-11T23:53:30,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369210585, max timestamp=1731369210623Num uniques 859; 2024-11-11T23:53:30,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:30,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:30,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:30,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369270735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:30,797 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/2107f983dc4b46b390458390f64ca911 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2107f983dc4b46b390458390f64ca911 2024-11-11T23:53:30,821 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into 2107f983dc4b46b390458390f64ca911(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:30,821 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:30,821 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369210316; duration=0sec 2024-11-11T23:53:30,821 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:30,821 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:30,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369270845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:30,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/cc2be082a02b4477a7614ac49b42886d 2024-11-11T23:53:30,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/1dbe0334ab884ec0a7a767e5a0ce3abb is 50, key is test_row_0/B:col10/1731369210489/Put/seqid=0 2024-11-11T23:53:30,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741859_1035 (size=12101) 2024-11-11T23:53:31,037 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T23:53:31,038 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T23:53:31,039 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T23:53:31,040 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T23:53:31,040 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T23:53:31,040 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T23:53:31,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T23:53:31,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T23:53:31,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369271055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369271205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369271205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:31,214 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:31,214 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:31,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369271235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:31,243 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:31,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369271237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:31,250 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:53:31,275 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:31,275 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T23:53:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T23:53:31,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:31,282 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:31,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:31,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369271364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:31,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/1dbe0334ab884ec0a7a767e5a0ce3abb 2024-11-11T23:53:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T23:53:31,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/7e4d731e7f694be58106e3363bad9625 is 50, key is test_row_0/C:col10/1731369210489/Put/seqid=0 2024-11-11T23:53:31,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741860_1036 (size=12101) 2024-11-11T23:53:31,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T23:53:31,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:31,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T23:53:31,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T23:53:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:31,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T23:53:31,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:31,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:31,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:31,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/7e4d731e7f694be58106e3363bad9625 2024-11-11T23:53:31,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/cc2be082a02b4477a7614ac49b42886d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/cc2be082a02b4477a7614ac49b42886d 2024-11-11T23:53:31,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/cc2be082a02b4477a7614ac49b42886d, entries=200, sequenceid=134, filesize=14.1 K 2024-11-11T23:53:31,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/1dbe0334ab884ec0a7a767e5a0ce3abb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1dbe0334ab884ec0a7a767e5a0ce3abb 2024-11-11T23:53:31,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1dbe0334ab884ec0a7a767e5a0ce3abb, entries=150, sequenceid=134, filesize=11.8 K 2024-11-11T23:53:31,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/7e4d731e7f694be58106e3363bad9625 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/7e4d731e7f694be58106e3363bad9625 2024-11-11T23:53:31,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/7e4d731e7f694be58106e3363bad9625, entries=150, sequenceid=134, filesize=11.8 K 2024-11-11T23:53:31,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6b2061b8554b1e7422f6c037ae02cccd in 1368ms, sequenceid=134, compaction requested=false 2024-11-11T23:53:31,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:31,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:31,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T23:53:31,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:31,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:31,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:31,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:31,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:31,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:31,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/38df981668234d18bd0389650b2cc002 is 50, key is test_row_0/A:col10/1731369210732/Put/seqid=0 2024-11-11T23:53:31,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T23:53:31,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:31,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:31,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T23:53:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:31,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741861_1037 (size=14541) 2024-11-11T23:53:31,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/38df981668234d18bd0389650b2cc002 2024-11-11T23:53:31,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369271937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:31,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/aafbb821305747f9a2b46dda4073b90c is 50, key is test_row_0/B:col10/1731369210732/Put/seqid=0 2024-11-11T23:53:31,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741862_1038 (size=12151) 2024-11-11T23:53:32,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369272054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:32,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T23:53:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:32,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T23:53:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369272265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:32,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T23:53:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:32,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/aafbb821305747f9a2b46dda4073b90c 2024-11-11T23:53:32,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/01a3cccd39b544908cf48e9c4a0b73fa is 50, key is test_row_0/C:col10/1731369210732/Put/seqid=0 2024-11-11T23:53:32,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741863_1039 (size=12151) 2024-11-11T23:53:32,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/01a3cccd39b544908cf48e9c4a0b73fa 2024-11-11T23:53:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T23:53:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/38df981668234d18bd0389650b2cc002 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/38df981668234d18bd0389650b2cc002 2024-11-11T23:53:32,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/38df981668234d18bd0389650b2cc002, entries=200, sequenceid=172, filesize=14.2 K 2024-11-11T23:53:32,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/aafbb821305747f9a2b46dda4073b90c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/aafbb821305747f9a2b46dda4073b90c 2024-11-11T23:53:32,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/aafbb821305747f9a2b46dda4073b90c, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T23:53:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/01a3cccd39b544908cf48e9c4a0b73fa as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/01a3cccd39b544908cf48e9c4a0b73fa 2024-11-11T23:53:32,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/01a3cccd39b544908cf48e9c4a0b73fa, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T23:53:32,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6b2061b8554b1e7422f6c037ae02cccd in 580ms, sequenceid=172, compaction requested=true 2024-11-11T23:53:32,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:32,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:32,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:32,468 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:32,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:32,468 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:32,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:32,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:32,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:32,470 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:32,470 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:32,471 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,471 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2107f983dc4b46b390458390f64ca911, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1dbe0334ab884ec0a7a767e5a0ce3abb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/aafbb821305747f9a2b46dda4073b90c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.6 K 2024-11-11T23:53:32,473 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2107f983dc4b46b390458390f64ca911, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731369209021 2024-11-11T23:53:32,473 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41189 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:32,474 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:32,474 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,474 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/573d948c57124e3cb52511d955ab6e90, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/cc2be082a02b4477a7614ac49b42886d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/38df981668234d18bd0389650b2cc002] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=40.2 K 2024-11-11T23:53:32,475 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dbe0334ab884ec0a7a767e5a0ce3abb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731369210133 2024-11-11T23:53:32,475 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting aafbb821305747f9a2b46dda4073b90c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731369210709 2024-11-11T23:53:32,475 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 573d948c57124e3cb52511d955ab6e90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731369209021 2024-11-11T23:53:32,477 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc2be082a02b4477a7614ac49b42886d, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731369210117 2024-11-11T23:53:32,478 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38df981668234d18bd0389650b2cc002, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731369210709 2024-11-11T23:53:32,504 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:32,504 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/dc508db133f446c6a389748e08a8738c is 50, key is test_row_0/B:col10/1731369210732/Put/seqid=0 2024-11-11T23:53:32,519 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:32,520 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6c1f5ac330424205a79134beec7ea46d is 50, key is test_row_0/A:col10/1731369210732/Put/seqid=0 2024-11-11T23:53:32,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T23:53:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,526 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:53:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:32,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741864_1040 (size=12459) 2024-11-11T23:53:32,548 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/dc508db133f446c6a389748e08a8738c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/dc508db133f446c6a389748e08a8738c 2024-11-11T23:53:32,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/11cba5d8e6ed4737bd8a3d636abc82cd is 50, key is test_row_0/A:col10/1731369211925/Put/seqid=0 2024-11-11T23:53:32,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741865_1041 (size=12459) 2024-11-11T23:53:32,566 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into dc508db133f446c6a389748e08a8738c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:32,566 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:32,566 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369212468; duration=0sec 2024-11-11T23:53:32,567 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:32,567 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:32,567 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:32,570 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:32,570 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:32,570 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:32,570 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f3344eb4a9f34fbb9188aed49aa0aec9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/7e4d731e7f694be58106e3363bad9625, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/01a3cccd39b544908cf48e9c4a0b73fa] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.6 K 2024-11-11T23:53:32,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741866_1042 (size=12151) 2024-11-11T23:53:32,571 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f3344eb4a9f34fbb9188aed49aa0aec9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731369209021 2024-11-11T23:53:32,572 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4d731e7f694be58106e3363bad9625, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731369210133 2024-11-11T23:53:32,573 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 01a3cccd39b544908cf48e9c4a0b73fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731369210709 2024-11-11T23:53:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:32,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:32,576 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/11cba5d8e6ed4737bd8a3d636abc82cd 2024-11-11T23:53:32,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/1fef74c0c1fc4cd6b6602151cf9ac2d9 is 50, key is test_row_0/B:col10/1731369211925/Put/seqid=0 2024-11-11T23:53:32,615 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:32,616 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b08fa650b9a147cc8b03d97012352ff8 is 50, key is test_row_0/C:col10/1731369210732/Put/seqid=0 2024-11-11T23:53:32,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741868_1044 (size=12459) 2024-11-11T23:53:32,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741867_1043 (size=12151) 2024-11-11T23:53:32,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369212640, max timestamp=1731369212675Num uniques 859; 2024-11-11T23:53:32,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369212640, max timestamp=1731369212675Num uniques 859; 2024-11-11T23:53:32,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:32,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:32,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369212640, max timestamp=1731369212675Num uniques 859; 2024-11-11T23:53:32,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:32,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369272761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:32,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369272875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:32,965 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6c1f5ac330424205a79134beec7ea46d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6c1f5ac330424205a79134beec7ea46d 2024-11-11T23:53:32,974 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into 6c1f5ac330424205a79134beec7ea46d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:32,974 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:32,974 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369212468; duration=0sec 2024-11-11T23:53:32,974 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:32,975 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:33,047 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/1fef74c0c1fc4cd6b6602151cf9ac2d9 2024-11-11T23:53:33,050 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b08fa650b9a147cc8b03d97012352ff8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b08fa650b9a147cc8b03d97012352ff8 2024-11-11T23:53:33,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/67c65f3af81844cba2d65c5b82ba38fe is 50, key is test_row_0/C:col10/1731369211925/Put/seqid=0 2024-11-11T23:53:33,065 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into b08fa650b9a147cc8b03d97012352ff8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:33,066 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:33,066 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369212468; duration=0sec 2024-11-11T23:53:33,066 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:33,066 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:33,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741869_1045 (size=12151) 2024-11-11T23:53:33,073 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/67c65f3af81844cba2d65c5b82ba38fe 2024-11-11T23:53:33,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/11cba5d8e6ed4737bd8a3d636abc82cd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/11cba5d8e6ed4737bd8a3d636abc82cd 2024-11-11T23:53:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369273085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,099 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/11cba5d8e6ed4737bd8a3d636abc82cd, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T23:53:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/1fef74c0c1fc4cd6b6602151cf9ac2d9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1fef74c0c1fc4cd6b6602151cf9ac2d9 2024-11-11T23:53:33,115 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1fef74c0c1fc4cd6b6602151cf9ac2d9, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T23:53:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/67c65f3af81844cba2d65c5b82ba38fe as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/67c65f3af81844cba2d65c5b82ba38fe 2024-11-11T23:53:33,125 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/67c65f3af81844cba2d65c5b82ba38fe, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T23:53:33,126 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6b2061b8554b1e7422f6c037ae02cccd in 601ms, sequenceid=182, compaction requested=false 2024-11-11T23:53:33,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T23:53:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T23:53:33,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T23:53:33,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8470 sec 2024-11-11T23:53:33,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8610 sec 2024-11-11T23:53:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:33,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T23:53:33,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:33,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:33,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:33,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:33,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:33,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:33,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/43807b914896481b917c6bcc1c4d09e0 is 50, key is test_row_0/A:col10/1731369213246/Put/seqid=0 2024-11-11T23:53:33,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741870_1046 (size=12151) 2024-11-11T23:53:33,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369273269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369273269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369273271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369273271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369273384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369273384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369273385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369273385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369273394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T23:53:33,425 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:33,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T23:53:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T23:53:33,428 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:33,429 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:33,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T23:53:33,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T23:53:33,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:33,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:33,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:33,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369273594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369273594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369273594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369273595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/43807b914896481b917c6bcc1c4d09e0 2024-11-11T23:53:33,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/5b1eedd67850433b9055442a2d742bb9 is 50, key is test_row_0/B:col10/1731369213246/Put/seqid=0 2024-11-11T23:53:33,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741871_1047 (size=12151) 2024-11-11T23:53:33,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T23:53:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:33,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T23:53:33,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T23:53:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:33,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:33,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369273904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369273904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369273905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369273907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:33,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369273914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T23:53:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:34,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T23:53:34,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/5b1eedd67850433b9055442a2d742bb9 2024-11-11T23:53:34,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/59b8a591c63a41b5951995039b982465 is 50, key is test_row_0/C:col10/1731369213246/Put/seqid=0 2024-11-11T23:53:34,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741872_1048 (size=12151) 2024-11-11T23:53:34,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/59b8a591c63a41b5951995039b982465 2024-11-11T23:53:34,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/43807b914896481b917c6bcc1c4d09e0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/43807b914896481b917c6bcc1c4d09e0 2024-11-11T23:53:34,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/43807b914896481b917c6bcc1c4d09e0, entries=150, sequenceid=222, filesize=11.9 K 2024-11-11T23:53:34,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/5b1eedd67850433b9055442a2d742bb9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5b1eedd67850433b9055442a2d742bb9 2024-11-11T23:53:34,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5b1eedd67850433b9055442a2d742bb9, entries=150, sequenceid=222, filesize=11.9 K 2024-11-11T23:53:34,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/59b8a591c63a41b5951995039b982465 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/59b8a591c63a41b5951995039b982465 2024-11-11T23:53:34,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T23:53:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:34,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:34,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/59b8a591c63a41b5951995039b982465, entries=150, sequenceid=222, filesize=11.9 K 2024-11-11T23:53:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:34,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 6b2061b8554b1e7422f6c037ae02cccd in 961ms, sequenceid=222, compaction requested=true 2024-11-11T23:53:34,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:34,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:34,209 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:34,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:34,210 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:34,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:34,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:34,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:34,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:34,213 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:34,214 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:34,214 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:34,214 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6c1f5ac330424205a79134beec7ea46d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/11cba5d8e6ed4737bd8a3d636abc82cd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/43807b914896481b917c6bcc1c4d09e0] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.9 K 2024-11-11T23:53:34,215 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c1f5ac330424205a79134beec7ea46d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731369210709 2024-11-11T23:53:34,216 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:34,216 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11cba5d8e6ed4737bd8a3d636abc82cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731369211902 2024-11-11T23:53:34,216 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:34,216 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:34,217 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/dc508db133f446c6a389748e08a8738c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1fef74c0c1fc4cd6b6602151cf9ac2d9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5b1eedd67850433b9055442a2d742bb9] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.9 K 2024-11-11T23:53:34,224 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting dc508db133f446c6a389748e08a8738c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731369210709 2024-11-11T23:53:34,224 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43807b914896481b917c6bcc1c4d09e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731369212745 2024-11-11T23:53:34,225 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fef74c0c1fc4cd6b6602151cf9ac2d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731369211902 2024-11-11T23:53:34,226 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1eedd67850433b9055442a2d742bb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731369212745 2024-11-11T23:53:34,252 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:34,253 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/fdc420a835a3471da425b8c34f4e8b3f is 50, key is test_row_0/A:col10/1731369213246/Put/seqid=0 2024-11-11T23:53:34,260 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:34,261 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/ab466e8dc4e34ae7aed145c2c53b9c60 is 50, key is test_row_0/B:col10/1731369213246/Put/seqid=0 2024-11-11T23:53:34,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741873_1049 (size=12561) 2024-11-11T23:53:34,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741874_1050 (size=12561) 2024-11-11T23:53:34,305 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/ab466e8dc4e34ae7aed145c2c53b9c60 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ab466e8dc4e34ae7aed145c2c53b9c60 2024-11-11T23:53:34,316 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into ab466e8dc4e34ae7aed145c2c53b9c60(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:34,316 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:34,316 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369214210; duration=0sec 2024-11-11T23:53:34,316 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:34,316 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:34,316 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:34,319 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:34,319 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:34,319 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:34,319 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b08fa650b9a147cc8b03d97012352ff8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/67c65f3af81844cba2d65c5b82ba38fe, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/59b8a591c63a41b5951995039b982465] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=35.9 K 2024-11-11T23:53:34,320 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b08fa650b9a147cc8b03d97012352ff8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731369210709 2024-11-11T23:53:34,320 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c65f3af81844cba2d65c5b82ba38fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731369211902 2024-11-11T23:53:34,320 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b8a591c63a41b5951995039b982465, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731369212745 2024-11-11T23:53:34,339 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:34,340 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/04b677710f504598ae5d798016dd161c is 50, key is test_row_0/C:col10/1731369213246/Put/seqid=0 2024-11-11T23:53:34,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T23:53:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:34,360 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:34,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741875_1051 (size=12561) 2024-11-11T23:53:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6f03ba982f7240bfb00a5241578f09e1 is 50, key is test_row_0/A:col10/1731369213270/Put/seqid=0 2024-11-11T23:53:34,385 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/04b677710f504598ae5d798016dd161c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/04b677710f504598ae5d798016dd161c 2024-11-11T23:53:34,404 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into 04b677710f504598ae5d798016dd161c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:34,404 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:34,404 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369214210; duration=0sec 2024-11-11T23:53:34,404 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:34,405 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:34,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741876_1052 (size=12151) 2024-11-11T23:53:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:34,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:34,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369214474, max timestamp=1731369214494Num uniques 859; 2024-11-11T23:53:34,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369214474, max timestamp=1731369214494Num uniques 859; 2024-11-11T23:53:34,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:34,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:34,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731369214474, max timestamp=1731369214494Num uniques 859; 2024-11-11T23:53:34,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T23:53:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369274534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369274534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369274541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369274543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T23:53:34,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369274655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369274655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369274655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369274655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,693 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/fdc420a835a3471da425b8c34f4e8b3f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/fdc420a835a3471da425b8c34f4e8b3f 2024-11-11T23:53:34,712 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into fdc420a835a3471da425b8c34f4e8b3f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:34,712 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:34,712 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369214209; duration=0sec 2024-11-11T23:53:34,713 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:34,713 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:34,811 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6f03ba982f7240bfb00a5241578f09e1 2024-11-11T23:53:34,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/5930e33ecfca4354a873d70bed91458b is 50, key is test_row_0/B:col10/1731369213270/Put/seqid=0 2024-11-11T23:53:34,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741877_1053 (size=12151) 2024-11-11T23:53:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369274865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369274865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369274866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369274865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:34,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369274925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:34,926 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369275174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369275175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:35,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369275175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:35,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369275175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,266 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/5930e33ecfca4354a873d70bed91458b 2024-11-11T23:53:35,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/307ec42f5e6b4f529603ef4df3246085 is 50, key is test_row_0/C:col10/1731369213270/Put/seqid=0 2024-11-11T23:53:35,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741878_1054 (size=12151) 2024-11-11T23:53:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T23:53:35,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:35,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369275685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369275685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369275685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369275694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:35,732 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/307ec42f5e6b4f529603ef4df3246085 2024-11-11T23:53:35,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6f03ba982f7240bfb00a5241578f09e1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6f03ba982f7240bfb00a5241578f09e1 2024-11-11T23:53:35,748 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6f03ba982f7240bfb00a5241578f09e1, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T23:53:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/5930e33ecfca4354a873d70bed91458b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5930e33ecfca4354a873d70bed91458b 2024-11-11T23:53:35,756 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5930e33ecfca4354a873d70bed91458b, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T23:53:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/307ec42f5e6b4f529603ef4df3246085 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/307ec42f5e6b4f529603ef4df3246085 2024-11-11T23:53:35,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/307ec42f5e6b4f529603ef4df3246085, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T23:53:35,766 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 6b2061b8554b1e7422f6c037ae02cccd in 1406ms, sequenceid=233, compaction requested=false 2024-11-11T23:53:35,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:35,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:35,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T23:53:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T23:53:35,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T23:53:35,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3400 sec 2024-11-11T23:53:35,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.3470 sec 2024-11-11T23:53:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:36,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=140.49 KB heapSize=404.70 KB 2024-11-11T23:53:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:53:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:53:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:53:36,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/a5a510320d6f4cb3a630af7ffc0b06d2 is 50, key is test_row_0/A:col10/1731369214538/Put/seqid=0 2024-11-11T23:53:36,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741879_1055 (size=14741) 2024-11-11T23:53:36,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/a5a510320d6f4cb3a630af7ffc0b06d2 2024-11-11T23:53:36,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369276712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:36,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369276715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:36,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369276716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:36,721 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369276717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:36,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:36,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/ad7fcc1fae6f49bb90998e5780a20b8c is 50, key is test_row_0/B:col10/1731369214538/Put/seqid=0 2024-11-11T23:53:36,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741880_1056 (size=12301) 2024-11-11T23:53:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369276825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369276825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369276825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:36,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369276954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:36,958 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369277034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369277034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369277034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/ad7fcc1fae6f49bb90998e5780a20b8c 2024-11-11T23:53:37,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/41b6e30cb2a143e0a82535273771e3e4 is 50, key is test_row_0/C:col10/1731369214538/Put/seqid=0 2024-11-11T23:53:37,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741881_1057 (size=12301) 2024-11-11T23:53:37,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/41b6e30cb2a143e0a82535273771e3e4 2024-11-11T23:53:37,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/a5a510320d6f4cb3a630af7ffc0b06d2 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a5a510320d6f4cb3a630af7ffc0b06d2 2024-11-11T23:53:37,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a5a510320d6f4cb3a630af7ffc0b06d2, entries=200, sequenceid=273, filesize=14.4 K 2024-11-11T23:53:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/ad7fcc1fae6f49bb90998e5780a20b8c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ad7fcc1fae6f49bb90998e5780a20b8c 2024-11-11T23:53:37,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ad7fcc1fae6f49bb90998e5780a20b8c, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T23:53:37,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/41b6e30cb2a143e0a82535273771e3e4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/41b6e30cb2a143e0a82535273771e3e4 2024-11-11T23:53:37,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/41b6e30cb2a143e0a82535273771e3e4, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T23:53:37,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~404.65 KB/414360, currentSize=40.25 KB/41220 for 6b2061b8554b1e7422f6c037ae02cccd in 504ms, sequenceid=273, compaction requested=true 2024-11-11T23:53:37,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:37,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:37,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:37,203 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:37,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:37,203 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:37,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:37,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:37,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:37,205 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:37,205 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:37,205 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:37,205 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:37,205 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:37,205 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:37,205 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/fdc420a835a3471da425b8c34f4e8b3f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6f03ba982f7240bfb00a5241578f09e1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a5a510320d6f4cb3a630af7ffc0b06d2] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=38.5 K 2024-11-11T23:53:37,205 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ab466e8dc4e34ae7aed145c2c53b9c60, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5930e33ecfca4354a873d70bed91458b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ad7fcc1fae6f49bb90998e5780a20b8c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=36.1 K 2024-11-11T23:53:37,206 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc420a835a3471da425b8c34f4e8b3f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731369212745 2024-11-11T23:53:37,206 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ab466e8dc4e34ae7aed145c2c53b9c60, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731369212745 2024-11-11T23:53:37,206 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f03ba982f7240bfb00a5241578f09e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731369213265 2024-11-11T23:53:37,206 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5930e33ecfca4354a873d70bed91458b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731369213265 2024-11-11T23:53:37,207 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ad7fcc1fae6f49bb90998e5780a20b8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731369214531 2024-11-11T23:53:37,207 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5a510320d6f4cb3a630af7ffc0b06d2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731369214531 2024-11-11T23:53:37,223 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:37,223 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#45 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:37,223 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6b34901033204271a219cf22f16e3fe1 is 50, key is test_row_0/A:col10/1731369214538/Put/seqid=0 2024-11-11T23:53:37,224 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/2b4049a99ad84e9588ccb5db3d6bae0a is 50, key is test_row_0/B:col10/1731369214538/Put/seqid=0 2024-11-11T23:53:37,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741882_1058 (size=12813) 2024-11-11T23:53:37,256 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/2b4049a99ad84e9588ccb5db3d6bae0a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2b4049a99ad84e9588ccb5db3d6bae0a 2024-11-11T23:53:37,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741883_1059 (size=12813) 2024-11-11T23:53:37,274 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into 2b4049a99ad84e9588ccb5db3d6bae0a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:37,274 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:37,274 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369217203; duration=0sec 2024-11-11T23:53:37,274 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:37,274 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:37,274 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:37,276 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:37,277 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:37,277 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:37,277 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/04b677710f504598ae5d798016dd161c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/307ec42f5e6b4f529603ef4df3246085, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/41b6e30cb2a143e0a82535273771e3e4] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=36.1 K 2024-11-11T23:53:37,278 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b677710f504598ae5d798016dd161c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731369212745 2024-11-11T23:53:37,278 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 307ec42f5e6b4f529603ef4df3246085, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731369213265 2024-11-11T23:53:37,279 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b6e30cb2a143e0a82535273771e3e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731369214531 2024-11-11T23:53:37,293 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#47 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:37,294 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/2afddd48c36344ceb82e4f5a3cddaac3 is 50, key is test_row_0/C:col10/1731369214538/Put/seqid=0 2024-11-11T23:53:37,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741884_1060 (size=12813) 2024-11-11T23:53:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:37,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:53:37,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:37,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/3086d9aa63134785b670be74384e86b9 is 50, key is test_row_0/A:col10/1731369217347/Put/seqid=0 2024-11-11T23:53:37,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741885_1061 (size=17181) 2024-11-11T23:53:37,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/3086d9aa63134785b670be74384e86b9 2024-11-11T23:53:37,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a1f7e907d3aa4803a284a13b919921c0 is 50, key is test_row_0/B:col10/1731369217347/Put/seqid=0 2024-11-11T23:53:37,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369277408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369277410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741886_1062 (size=12301) 2024-11-11T23:53:37,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a1f7e907d3aa4803a284a13b919921c0 2024-11-11T23:53:37,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369277414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b20c8b3bf48d46da930ca464eeb6c74e is 50, key is test_row_0/C:col10/1731369217347/Put/seqid=0 2024-11-11T23:53:37,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741887_1063 (size=12301) 2024-11-11T23:53:37,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369277524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369277525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369277525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T23:53:37,595 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:37,595 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T23:53:37,597 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T23:53:37,598 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:37,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:37,678 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6b34901033204271a219cf22f16e3fe1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b34901033204271a219cf22f16e3fe1 2024-11-11T23:53:37,684 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into 6b34901033204271a219cf22f16e3fe1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:37,684 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:37,685 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369217203; duration=0sec 2024-11-11T23:53:37,685 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:37,685 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T23:53:37,712 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/2afddd48c36344ceb82e4f5a3cddaac3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2afddd48c36344ceb82e4f5a3cddaac3 2024-11-11T23:53:37,722 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into 2afddd48c36344ceb82e4f5a3cddaac3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:37,722 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:37,722 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369217203; duration=0sec 2024-11-11T23:53:37,722 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:37,722 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369277744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369277744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369277744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:37,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T23:53:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:37,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:37,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b20c8b3bf48d46da930ca464eeb6c74e 2024-11-11T23:53:37,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/3086d9aa63134785b670be74384e86b9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/3086d9aa63134785b670be74384e86b9 2024-11-11T23:53:37,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/3086d9aa63134785b670be74384e86b9, entries=250, sequenceid=286, filesize=16.8 K 2024-11-11T23:53:37,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a1f7e907d3aa4803a284a13b919921c0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a1f7e907d3aa4803a284a13b919921c0 2024-11-11T23:53:37,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a1f7e907d3aa4803a284a13b919921c0, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T23:53:37,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b20c8b3bf48d46da930ca464eeb6c74e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b20c8b3bf48d46da930ca464eeb6c74e 2024-11-11T23:53:37,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b20c8b3bf48d46da930ca464eeb6c74e, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T23:53:37,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6b2061b8554b1e7422f6c037ae02cccd in 520ms, sequenceid=286, compaction requested=false 2024-11-11T23:53:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:37,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T23:53:37,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:37,905 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T23:53:37,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:37,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:37,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:37,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:37,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:37,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/34ffb723bf68418da04dc5b5e7c59578 is 50, key is test_row_0/A:col10/1731369217407/Put/seqid=0 2024-11-11T23:53:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T23:53:37,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741888_1064 (size=12301) 2024-11-11T23:53:37,919 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/34ffb723bf68418da04dc5b5e7c59578 2024-11-11T23:53:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/4e1deb74ec834d22a72e5de784651664 is 50, key is test_row_0/B:col10/1731369217407/Put/seqid=0 2024-11-11T23:53:37,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741889_1065 (size=12301) 2024-11-11T23:53:37,944 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/4e1deb74ec834d22a72e5de784651664 2024-11-11T23:53:37,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/fe09cdc9a530419bb0289f3af8c4bb9a is 50, key is test_row_0/C:col10/1731369217407/Put/seqid=0 2024-11-11T23:53:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741890_1066 (size=12301) 2024-11-11T23:53:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:38,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369278068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369278069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369278071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369278185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369278185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369278185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T23:53:38,371 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/fe09cdc9a530419bb0289f3af8c4bb9a 2024-11-11T23:53:38,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/34ffb723bf68418da04dc5b5e7c59578 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/34ffb723bf68418da04dc5b5e7c59578 2024-11-11T23:53:38,386 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/34ffb723bf68418da04dc5b5e7c59578, entries=150, sequenceid=308, filesize=12.0 K 2024-11-11T23:53:38,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/4e1deb74ec834d22a72e5de784651664 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4e1deb74ec834d22a72e5de784651664 2024-11-11T23:53:38,394 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4e1deb74ec834d22a72e5de784651664, entries=150, sequenceid=308, filesize=12.0 K 2024-11-11T23:53:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/fe09cdc9a530419bb0289f3af8c4bb9a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/fe09cdc9a530419bb0289f3af8c4bb9a 2024-11-11T23:53:38,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369278394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369278394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369278394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/fe09cdc9a530419bb0289f3af8c4bb9a, entries=150, sequenceid=308, filesize=12.0 K 2024-11-11T23:53:38,411 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6b2061b8554b1e7422f6c037ae02cccd in 506ms, sequenceid=308, compaction requested=true 2024-11-11T23:53:38,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:38,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:38,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T23:53:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T23:53:38,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T23:53:38,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 816 msec 2024-11-11T23:53:38,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 823 msec 2024-11-11T23:53:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:38,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T23:53:38,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:38,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:38,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:38,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:38,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:38,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:38,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/d32abe9da26f49ba8f6e8420fa57859c is 50, key is test_row_0/A:col10/1731369218066/Put/seqid=0 2024-11-11T23:53:38,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741891_1067 (size=17181) 2024-11-11T23:53:38,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/d32abe9da26f49ba8f6e8420fa57859c 2024-11-11T23:53:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T23:53:38,735 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:38,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T23:53:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T23:53:38,741 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:38,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/4f8837680ea14e26ab9911fe758e42ac is 50, key is test_row_0/B:col10/1731369218066/Put/seqid=0 2024-11-11T23:53:38,742 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:38,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:38,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741892_1068 (size=12301) 2024-11-11T23:53:38,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/4f8837680ea14e26ab9911fe758e42ac 2024-11-11T23:53:38,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/3398ec2f6cea4f4084dd86863b966d8b is 50, key is test_row_0/C:col10/1731369218066/Put/seqid=0 2024-11-11T23:53:38,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741893_1069 (size=12301) 2024-11-11T23:53:38,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/3398ec2f6cea4f4084dd86863b966d8b 2024-11-11T23:53:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369278759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369278762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369278759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369278762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/d32abe9da26f49ba8f6e8420fa57859c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/d32abe9da26f49ba8f6e8420fa57859c 2024-11-11T23:53:38,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/d32abe9da26f49ba8f6e8420fa57859c, entries=250, sequenceid=322, filesize=16.8 K 2024-11-11T23:53:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/4f8837680ea14e26ab9911fe758e42ac as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4f8837680ea14e26ab9911fe758e42ac 2024-11-11T23:53:38,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4f8837680ea14e26ab9911fe758e42ac, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T23:53:38,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/3398ec2f6cea4f4084dd86863b966d8b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/3398ec2f6cea4f4084dd86863b966d8b 2024-11-11T23:53:38,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/3398ec2f6cea4f4084dd86863b966d8b, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T23:53:38,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 6b2061b8554b1e7422f6c037ae02cccd in 111ms, sequenceid=322, compaction requested=true 2024-11-11T23:53:38,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:38,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:38,819 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:53:38,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:38,819 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:53:38,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:38,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:38,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:38,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:38,826 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:53:38,826 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:38,826 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:38,826 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2b4049a99ad84e9588ccb5db3d6bae0a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a1f7e907d3aa4803a284a13b919921c0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4e1deb74ec834d22a72e5de784651664, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4f8837680ea14e26ab9911fe758e42ac] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=48.6 K 2024-11-11T23:53:38,827 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:53:38,827 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:38,827 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:38,827 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b34901033204271a219cf22f16e3fe1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/3086d9aa63134785b670be74384e86b9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/34ffb723bf68418da04dc5b5e7c59578, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/d32abe9da26f49ba8f6e8420fa57859c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=58.1 K 2024-11-11T23:53:38,828 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b4049a99ad84e9588ccb5db3d6bae0a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731369214531 2024-11-11T23:53:38,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b34901033204271a219cf22f16e3fe1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731369214531 2024-11-11T23:53:38,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3086d9aa63134785b670be74384e86b9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731369216714 2024-11-11T23:53:38,828 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f7e907d3aa4803a284a13b919921c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731369216714 2024-11-11T23:53:38,829 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34ffb723bf68418da04dc5b5e7c59578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731369217391 2024-11-11T23:53:38,829 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1deb74ec834d22a72e5de784651664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731369217391 2024-11-11T23:53:38,829 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8837680ea14e26ab9911fe758e42ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731369218066 2024-11-11T23:53:38,829 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32abe9da26f49ba8f6e8420fa57859c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731369218065 2024-11-11T23:53:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T23:53:38,853 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#57 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:38,854 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7916a5449ccb4abea2157dab410990e5 is 50, key is test_row_0/A:col10/1731369218066/Put/seqid=0 2024-11-11T23:53:38,863 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#58 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:38,864 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/f6bff28adf5847f6aaa087cdcaf92bf6 is 50, key is test_row_0/B:col10/1731369218066/Put/seqid=0 2024-11-11T23:53:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741894_1070 (size=12949) 2024-11-11T23:53:38,893 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7916a5449ccb4abea2157dab410990e5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7916a5449ccb4abea2157dab410990e5 2024-11-11T23:53:38,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T23:53:38,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:38,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T23:53:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:38,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/263f68605a684a79ac5b6970cfc8ae55 is 50, key is test_row_0/A:col10/1731369218757/Put/seqid=0 2024-11-11T23:53:38,908 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into 7916a5449ccb4abea2157dab410990e5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:38,908 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:38,908 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=12, startTime=1731369218819; duration=0sec 2024-11-11T23:53:38,908 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:38,908 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:38,908 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:53:38,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741895_1071 (size=12949) 2024-11-11T23:53:38,912 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:53:38,912 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:38,912 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:38,914 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2afddd48c36344ceb82e4f5a3cddaac3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b20c8b3bf48d46da930ca464eeb6c74e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/fe09cdc9a530419bb0289f3af8c4bb9a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/3398ec2f6cea4f4084dd86863b966d8b] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=48.6 K 2024-11-11T23:53:38,915 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2afddd48c36344ceb82e4f5a3cddaac3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731369214531 2024-11-11T23:53:38,916 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b20c8b3bf48d46da930ca464eeb6c74e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731369216714 2024-11-11T23:53:38,918 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe09cdc9a530419bb0289f3af8c4bb9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731369217391 2024-11-11T23:53:38,918 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3398ec2f6cea4f4084dd86863b966d8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731369218066 2024-11-11T23:53:38,920 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/f6bff28adf5847f6aaa087cdcaf92bf6 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f6bff28adf5847f6aaa087cdcaf92bf6 2024-11-11T23:53:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369278914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369278916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369278927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369278928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:38,938 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into f6bff28adf5847f6aaa087cdcaf92bf6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:38,938 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:38,938 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=12, startTime=1731369218819; duration=0sec 2024-11-11T23:53:38,939 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:38,939 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:38,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741896_1072 (size=12301) 2024-11-11T23:53:38,948 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#60 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:38,949 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/74de01ab7d704b5e9b7af521d081190e is 50, key is test_row_0/C:col10/1731369218066/Put/seqid=0 2024-11-11T23:53:38,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741897_1073 (size=12949) 2024-11-11T23:53:38,978 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/74de01ab7d704b5e9b7af521d081190e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/74de01ab7d704b5e9b7af521d081190e 2024-11-11T23:53:38,993 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into 74de01ab7d704b5e9b7af521d081190e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:38,993 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:38,993 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=12, startTime=1731369218821; duration=0sec 2024-11-11T23:53:38,994 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:38,994 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:39,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369279034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369279035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369279035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369279044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T23:53:39,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369279244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369279244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369279244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369279255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,346 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/263f68605a684a79ac5b6970cfc8ae55 2024-11-11T23:53:39,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/302c299ffdfa4fbba7e7c259092b2de2 is 50, key is test_row_0/B:col10/1731369218757/Put/seqid=0 2024-11-11T23:53:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T23:53:39,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741898_1074 (size=12301) 2024-11-11T23:53:39,376 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/302c299ffdfa4fbba7e7c259092b2de2 2024-11-11T23:53:39,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/86b1bda9db024e30a12a2d317fa830ad is 50, key is test_row_0/C:col10/1731369218757/Put/seqid=0 2024-11-11T23:53:39,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741899_1075 (size=12301) 2024-11-11T23:53:39,411 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/86b1bda9db024e30a12a2d317fa830ad 2024-11-11T23:53:39,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/263f68605a684a79ac5b6970cfc8ae55 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/263f68605a684a79ac5b6970cfc8ae55 2024-11-11T23:53:39,430 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/263f68605a684a79ac5b6970cfc8ae55, entries=150, sequenceid=342, filesize=12.0 K 2024-11-11T23:53:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/302c299ffdfa4fbba7e7c259092b2de2 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/302c299ffdfa4fbba7e7c259092b2de2 2024-11-11T23:53:39,441 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/302c299ffdfa4fbba7e7c259092b2de2, entries=150, sequenceid=342, filesize=12.0 K 2024-11-11T23:53:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/86b1bda9db024e30a12a2d317fa830ad as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/86b1bda9db024e30a12a2d317fa830ad 2024-11-11T23:53:39,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/86b1bda9db024e30a12a2d317fa830ad, entries=150, sequenceid=342, filesize=12.0 K 2024-11-11T23:53:39,455 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6b2061b8554b1e7422f6c037ae02cccd in 560ms, sequenceid=342, compaction requested=false 2024-11-11T23:53:39,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:39,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:39,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T23:53:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T23:53:39,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T23:53:39,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 716 msec 2024-11-11T23:53:39,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 732 msec 2024-11-11T23:53:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:39,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T23:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:39,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/de0e1f0d15fd4aeb9963e669a5675b79 is 50, key is test_row_0/A:col10/1731369218923/Put/seqid=0 2024-11-11T23:53:39,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741900_1076 (size=14741) 2024-11-11T23:53:39,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/de0e1f0d15fd4aeb9963e669a5675b79 2024-11-11T23:53:39,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369279592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369279593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369279595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369279603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/64018c4ec52846dc8482c756da0660a8 is 50, key is test_row_0/B:col10/1731369218923/Put/seqid=0 2024-11-11T23:53:39,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741901_1077 (size=12301) 2024-11-11T23:53:39,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369279715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369279715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369279715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369279724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T23:53:39,875 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:39,876 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T23:53:39,878 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:39,880 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:39,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T23:53:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369279924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369279924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369279925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369279935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T23:53:40,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/64018c4ec52846dc8482c756da0660a8 2024-11-11T23:53:40,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T23:53:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/582703095e274533bdf71eca2686c7bc is 50, key is test_row_0/C:col10/1731369218923/Put/seqid=0 2024-11-11T23:53:40,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741902_1078 (size=12301) 2024-11-11T23:53:40,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T23:53:40,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:40,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T23:53:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369280235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369280235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369280235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369280244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T23:53:40,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:40,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:40,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/582703095e274533bdf71eca2686c7bc 2024-11-11T23:53:40,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/de0e1f0d15fd4aeb9963e669a5675b79 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/de0e1f0d15fd4aeb9963e669a5675b79 2024-11-11T23:53:40,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/de0e1f0d15fd4aeb9963e669a5675b79, entries=200, sequenceid=358, filesize=14.4 K 2024-11-11T23:53:40,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/64018c4ec52846dc8482c756da0660a8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/64018c4ec52846dc8482c756da0660a8 2024-11-11T23:53:40,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/64018c4ec52846dc8482c756da0660a8, entries=150, sequenceid=358, filesize=12.0 K 2024-11-11T23:53:40,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/582703095e274533bdf71eca2686c7bc as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/582703095e274533bdf71eca2686c7bc 2024-11-11T23:53:40,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/582703095e274533bdf71eca2686c7bc, entries=150, sequenceid=358, filesize=12.0 K 2024-11-11T23:53:40,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6b2061b8554b1e7422f6c037ae02cccd in 928ms, sequenceid=358, compaction requested=true 2024-11-11T23:53:40,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:40,487 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:40,487 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:40,488 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:40,488 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:40,488 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:40,488 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:40,488 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,488 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,488 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7916a5449ccb4abea2157dab410990e5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/263f68605a684a79ac5b6970cfc8ae55, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/de0e1f0d15fd4aeb9963e669a5675b79] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=39.1 K 2024-11-11T23:53:40,488 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f6bff28adf5847f6aaa087cdcaf92bf6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/302c299ffdfa4fbba7e7c259092b2de2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/64018c4ec52846dc8482c756da0660a8] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=36.7 K 2024-11-11T23:53:40,489 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7916a5449ccb4abea2157dab410990e5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731369218066 2024-11-11T23:53:40,489 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f6bff28adf5847f6aaa087cdcaf92bf6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731369218066 2024-11-11T23:53:40,489 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 263f68605a684a79ac5b6970cfc8ae55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731369218757 2024-11-11T23:53:40,489 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 302c299ffdfa4fbba7e7c259092b2de2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731369218757 2024-11-11T23:53:40,489 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting de0e1f0d15fd4aeb9963e669a5675b79, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731369218912 2024-11-11T23:53:40,489 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 64018c4ec52846dc8482c756da0660a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731369218912 2024-11-11T23:53:40,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T23:53:40,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T23:53:40,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:40,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:40,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:40,504 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:40,504 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:40,505 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/c86e6892385d46f6a7fbaf955593a5e5 is 50, key is test_row_0/B:col10/1731369218923/Put/seqid=0 2024-11-11T23:53:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T23:53:40,505 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/49a8a0af023d46cfa4cbe789dcd4db0a is 50, key is test_row_0/A:col10/1731369218923/Put/seqid=0 2024-11-11T23:53:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/020d267db3154818bc769ebc527a07b6 is 50, key is test_row_0/A:col10/1731369219600/Put/seqid=0 2024-11-11T23:53:40,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741903_1079 (size=13051) 2024-11-11T23:53:40,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741905_1081 (size=12301) 2024-11-11T23:53:40,528 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/020d267db3154818bc769ebc527a07b6 2024-11-11T23:53:40,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741904_1080 (size=13051) 2024-11-11T23:53:40,543 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/49a8a0af023d46cfa4cbe789dcd4db0a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/49a8a0af023d46cfa4cbe789dcd4db0a 2024-11-11T23:53:40,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/3dbc7c8e01c8479299bf3239bd150bc1 is 50, key is test_row_0/B:col10/1731369219600/Put/seqid=0 2024-11-11T23:53:40,554 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into 49a8a0af023d46cfa4cbe789dcd4db0a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:40,554 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:40,554 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369220486; duration=0sec 2024-11-11T23:53:40,555 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:40,555 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:40,555 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:40,556 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:40,556 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:40,556 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,556 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/74de01ab7d704b5e9b7af521d081190e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/86b1bda9db024e30a12a2d317fa830ad, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/582703095e274533bdf71eca2686c7bc] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=36.7 K 2024-11-11T23:53:40,557 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74de01ab7d704b5e9b7af521d081190e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731369218066 2024-11-11T23:53:40,558 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86b1bda9db024e30a12a2d317fa830ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731369218757 2024-11-11T23:53:40,558 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 582703095e274533bdf71eca2686c7bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731369218912 2024-11-11T23:53:40,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741906_1082 (size=12301) 2024-11-11T23:53:40,564 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/3dbc7c8e01c8479299bf3239bd150bc1 2024-11-11T23:53:40,580 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:40,581 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/dcfe811213c741f48514ca0775eea43a is 50, key is test_row_0/C:col10/1731369218923/Put/seqid=0 2024-11-11T23:53:40,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f is 50, key is test_row_0/C:col10/1731369219600/Put/seqid=0 2024-11-11T23:53:40,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741907_1083 (size=13051) 2024-11-11T23:53:40,613 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/dcfe811213c741f48514ca0775eea43a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/dcfe811213c741f48514ca0775eea43a 2024-11-11T23:53:40,626 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into dcfe811213c741f48514ca0775eea43a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:40,626 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:40,626 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369220487; duration=0sec 2024-11-11T23:53:40,626 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:40,626 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:40,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741908_1084 (size=12301) 2024-11-11T23:53:40,631 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f 2024-11-11T23:53:40,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/020d267db3154818bc769ebc527a07b6 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/020d267db3154818bc769ebc527a07b6 2024-11-11T23:53:40,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/020d267db3154818bc769ebc527a07b6, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T23:53:40,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/3dbc7c8e01c8479299bf3239bd150bc1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/3dbc7c8e01c8479299bf3239bd150bc1 2024-11-11T23:53:40,657 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/3dbc7c8e01c8479299bf3239bd150bc1, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T23:53:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f 2024-11-11T23:53:40,671 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T23:53:40,672 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 6b2061b8554b1e7422f6c037ae02cccd in 174ms, sequenceid=377, compaction requested=false 2024-11-11T23:53:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T23:53:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T23:53:40,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T23:53:40,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 794 msec 2024-11-11T23:53:40,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 802 msec 2024-11-11T23:53:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:40,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:53:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:40,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/492c8987e559430797ddc49bebd92c70 is 50, key is test_row_0/A:col10/1731369220754/Put/seqid=0 2024-11-11T23:53:40,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741909_1085 (size=14737) 2024-11-11T23:53:40,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/492c8987e559430797ddc49bebd92c70 2024-11-11T23:53:40,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/32332ee01f32437bac14364d2fd6546f is 50, key is test_row_0/B:col10/1731369220754/Put/seqid=0 2024-11-11T23:53:40,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741910_1086 (size=9857) 2024-11-11T23:53:40,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/32332ee01f32437bac14364d2fd6546f 2024-11-11T23:53:40,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/32cb142ab646458b912386bf3ede224a is 50, key is test_row_0/C:col10/1731369220754/Put/seqid=0 2024-11-11T23:53:40,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741911_1087 (size=9857) 2024-11-11T23:53:40,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/32cb142ab646458b912386bf3ede224a 2024-11-11T23:53:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369280835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369280836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369220828, max timestamp=1731369220836Num uniques 859; 2024-11-11T23:53:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369220828, max timestamp=1731369220836Num uniques 859; 2024-11-11T23:53:40,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:40,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369220828, max timestamp=1731369220836Num uniques 859; 2024-11-11T23:53:40,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/492c8987e559430797ddc49bebd92c70 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/492c8987e559430797ddc49bebd92c70 2024-11-11T23:53:40,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/492c8987e559430797ddc49bebd92c70, entries=200, sequenceid=390, filesize=14.4 K 2024-11-11T23:53:40,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/32332ee01f32437bac14364d2fd6546f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/32332ee01f32437bac14364d2fd6546f 2024-11-11T23:53:40,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/32332ee01f32437bac14364d2fd6546f, entries=100, sequenceid=390, filesize=9.6 K 2024-11-11T23:53:40,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/32cb142ab646458b912386bf3ede224a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/32cb142ab646458b912386bf3ede224a 2024-11-11T23:53:40,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/32cb142ab646458b912386bf3ede224a, entries=100, sequenceid=390, filesize=9.6 K 2024-11-11T23:53:40,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=107.34 KB/109920 for 6b2061b8554b1e7422f6c037ae02cccd in 129ms, sequenceid=390, compaction requested=true 2024-11-11T23:53:40,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:40,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:40,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:40,884 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:40,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:40,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:40,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:40,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T23:53:40,885 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:40,886 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:40,886 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,886 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/49a8a0af023d46cfa4cbe789dcd4db0a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/020d267db3154818bc769ebc527a07b6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/492c8987e559430797ddc49bebd92c70] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=39.1 K 2024-11-11T23:53:40,886 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49a8a0af023d46cfa4cbe789dcd4db0a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731369218912 2024-11-11T23:53:40,887 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 020d267db3154818bc769ebc527a07b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731369219566 2024-11-11T23:53:40,887 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492c8987e559430797ddc49bebd92c70, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369220748 2024-11-11T23:53:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:40,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=114.05 KB heapSize=325.22 KB 2024-11-11T23:53:40,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:40,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:40,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:40,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:40,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:40,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:40,897 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:40,898 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/4b63fdad760a4f219d79d2483d0e3f9b is 50, key is test_row_0/A:col10/1731369220754/Put/seqid=0 2024-11-11T23:53:40,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741912_1088 (size=13153) 2024-11-11T23:53:40,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/f8336b914daa4a20a4cae169f6bec3bb is 50, key is test_row_0/A:col10/1731369220882/Put/seqid=0 2024-11-11T23:53:40,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741913_1089 (size=12301) 2024-11-11T23:53:40,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369280926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,933 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/c86e6892385d46f6a7fbaf955593a5e5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c86e6892385d46f6a7fbaf955593a5e5 2024-11-11T23:53:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369280929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,950 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into c86e6892385d46f6a7fbaf955593a5e5(size=12.7 K), total size for store is 34.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:40,950 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:40,950 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369220487; duration=0sec 2024-11-11T23:53:40,950 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T23:53:40,951 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:40,951 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:40,951 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:40,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369280945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369280945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,954 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:40,954 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:40,954 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:40,954 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/dcfe811213c741f48514ca0775eea43a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/32cb142ab646458b912386bf3ede224a] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=34.4 K 2024-11-11T23:53:40,955 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting dcfe811213c741f48514ca0775eea43a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731369218912 2024-11-11T23:53:40,956 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b2bac6bf4bbb4e6daa5b73c25ebaa10f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731369219566 2024-11-11T23:53:40,956 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 32cb142ab646458b912386bf3ede224a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369220754 2024-11-11T23:53:40,972 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:40,973 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/6cc100116036467ebbec9916c17a9922 is 50, key is test_row_0/C:col10/1731369220754/Put/seqid=0 2024-11-11T23:53:40,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741914_1090 (size=13153) 2024-11-11T23:53:40,988 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/6cc100116036467ebbec9916c17a9922 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/6cc100116036467ebbec9916c17a9922 2024-11-11T23:53:40,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35690 deadline: 1731369280985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:40,990 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:40,996 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into 6cc100116036467ebbec9916c17a9922(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:40,996 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:40,996 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369220884; duration=0sec 2024-11-11T23:53:40,996 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:40,996 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:40,996 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:41,000 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:41,000 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:41,001 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,001 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c86e6892385d46f6a7fbaf955593a5e5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/3dbc7c8e01c8479299bf3239bd150bc1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/32332ee01f32437bac14364d2fd6546f] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=34.4 K 2024-11-11T23:53:41,001 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c86e6892385d46f6a7fbaf955593a5e5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731369218912 2024-11-11T23:53:41,001 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dbc7c8e01c8479299bf3239bd150bc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731369219566 2024-11-11T23:53:41,002 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 32332ee01f32437bac14364d2fd6546f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369220754 2024-11-11T23:53:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T23:53:41,015 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:41,016 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:41,017 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:41,018 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/8fc664f3cf9f41f8a7a3b03979aaf504 is 50, key is test_row_0/B:col10/1731369220754/Put/seqid=0 2024-11-11T23:53:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T23:53:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:53:41,025 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:41,026 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:41,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369281035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741915_1091 (size=13153) 2024-11-11T23:53:41,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369281044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,056 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/8fc664f3cf9f41f8a7a3b03979aaf504 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8fc664f3cf9f41f8a7a3b03979aaf504 2024-11-11T23:53:41,062 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into 8fc664f3cf9f41f8a7a3b03979aaf504(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:41,062 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:41,062 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369220884; duration=0sec 2024-11-11T23:53:41,063 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:41,063 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:53:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369281164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369281165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T23:53:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369281244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369281255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,318 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/4b63fdad760a4f219d79d2483d0e3f9b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4b63fdad760a4f219d79d2483d0e3f9b 2024-11-11T23:53:41,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/f8336b914daa4a20a4cae169f6bec3bb 2024-11-11T23:53:41,327 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into 4b63fdad760a4f219d79d2483d0e3f9b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:41,327 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:41,327 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369220884; duration=0sec 2024-11-11T23:53:41,327 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:41,327 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:41,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/b682d8f9bf554ba88b78dd51a436e363 is 50, key is test_row_0/B:col10/1731369220882/Put/seqid=0 2024-11-11T23:53:41,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T23:53:41,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:41,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741916_1092 (size=12301) 2024-11-11T23:53:41,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/b682d8f9bf554ba88b78dd51a436e363 2024-11-11T23:53:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:53:41,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/0076ef879f2c483e960636c03d381777 is 50, key is test_row_0/C:col10/1731369220882/Put/seqid=0 2024-11-11T23:53:41,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741917_1093 (size=12301) 2024-11-11T23:53:41,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369281475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369281475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T23:53:41,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:41,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369281565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369281565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:41,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:41,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T23:53:41,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:53:41,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/0076ef879f2c483e960636c03d381777 2024-11-11T23:53:41,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/f8336b914daa4a20a4cae169f6bec3bb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f8336b914daa4a20a4cae169f6bec3bb 2024-11-11T23:53:41,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f8336b914daa4a20a4cae169f6bec3bb, entries=150, sequenceid=424, filesize=12.0 K 2024-11-11T23:53:41,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/b682d8f9bf554ba88b78dd51a436e363 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b682d8f9bf554ba88b78dd51a436e363 2024-11-11T23:53:41,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b682d8f9bf554ba88b78dd51a436e363, entries=150, sequenceid=424, filesize=12.0 K 2024-11-11T23:53:41,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/0076ef879f2c483e960636c03d381777 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0076ef879f2c483e960636c03d381777 2024-11-11T23:53:41,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0076ef879f2c483e960636c03d381777, entries=150, sequenceid=424, filesize=12.0 K 2024-11-11T23:53:41,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~325.17 KB/332976, currentSize=67.09 KB/68700 for 6b2061b8554b1e7422f6c037ae02cccd in 892ms, sequenceid=424, compaction requested=false 2024-11-11T23:53:41,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:41,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T23:53:41,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:41,796 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T23:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:41,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7d2528726ec3436b90f67b5785c02985 is 50, key is test_row_0/A:col10/1731369220926/Put/seqid=0 2024-11-11T23:53:41,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741918_1094 (size=12301) 2024-11-11T23:53:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:41,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:42,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369282021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369282027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369282074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369282075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369282134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369282134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:53:42,210 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7d2528726ec3436b90f67b5785c02985 2024-11-11T23:53:42,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/edc6db057659442c85d64b1e401b32d5 is 50, key is test_row_0/B:col10/1731369220926/Put/seqid=0 2024-11-11T23:53:42,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741919_1095 (size=12301) 2024-11-11T23:53:42,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369282344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369282344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,623 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/edc6db057659442c85d64b1e401b32d5 2024-11-11T23:53:42,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/2e39cfd60ae6446a9ab983c4f038d6a3 is 50, key is test_row_0/C:col10/1731369220926/Put/seqid=0 2024-11-11T23:53:42,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741920_1096 (size=12301) 2024-11-11T23:53:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369282654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369282654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,038 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/2e39cfd60ae6446a9ab983c4f038d6a3 2024-11-11T23:53:43,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7d2528726ec3436b90f67b5785c02985 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7d2528726ec3436b90f67b5785c02985 2024-11-11T23:53:43,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7d2528726ec3436b90f67b5785c02985, entries=150, sequenceid=441, filesize=12.0 K 2024-11-11T23:53:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/edc6db057659442c85d64b1e401b32d5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/edc6db057659442c85d64b1e401b32d5 2024-11-11T23:53:43,055 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/edc6db057659442c85d64b1e401b32d5, entries=150, sequenceid=441, filesize=12.0 K 2024-11-11T23:53:43,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/2e39cfd60ae6446a9ab983c4f038d6a3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2e39cfd60ae6446a9ab983c4f038d6a3 2024-11-11T23:53:43,065 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2e39cfd60ae6446a9ab983c4f038d6a3, entries=150, sequenceid=441, filesize=12.0 K 2024-11-11T23:53:43,067 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6b2061b8554b1e7422f6c037ae02cccd in 1270ms, sequenceid=441, compaction requested=true 2024-11-11T23:53:43,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:43,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T23:53:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T23:53:43,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T23:53:43,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0430 sec 2024-11-11T23:53:43,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.0590 sec 2024-11-11T23:53:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:43,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T23:53:43,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:43,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:43,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:43,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:43,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:43,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:43,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7a0bf52f4e7d466db4915af3c5b58442 is 50, key is test_row_0/A:col10/1731369222025/Put/seqid=0 2024-11-11T23:53:43,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741921_1097 (size=14741) 2024-11-11T23:53:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369283121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369283125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369283165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369283165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:53:43,174 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:43,175 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T23:53:43,177 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T23:53:43,178 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:43,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:43,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369283235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369283235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T23:53:43,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T23:53:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369283444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369283445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T23:53:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T23:53:43,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7a0bf52f4e7d466db4915af3c5b58442 2024-11-11T23:53:43,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/39153066e2994972be2d278fc9407f14 is 50, key is test_row_0/B:col10/1731369222025/Put/seqid=0 2024-11-11T23:53:43,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741922_1098 (size=12301) 2024-11-11T23:53:43,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T23:53:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369283754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369283754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:43,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T23:53:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T23:53:43,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/39153066e2994972be2d278fc9407f14 2024-11-11T23:53:43,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/f22a8a760e0e4a4b9fa9208a9683ac82 is 50, key is test_row_0/C:col10/1731369222025/Put/seqid=0 2024-11-11T23:53:43,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T23:53:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:43,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:43,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:43,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741923_1099 (size=12301) 2024-11-11T23:53:44,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T23:53:44,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:44,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:44,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:44,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:44,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:44,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369284175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:44,175 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:44,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369284185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:44,188 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 6b2061b8554b1e7422f6c037ae02cccd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:44,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T23:53:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:44,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:44,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369284265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:44,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369284265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T23:53:44,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/f22a8a760e0e4a4b9fa9208a9683ac82 2024-11-11T23:53:44,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/7a0bf52f4e7d466db4915af3c5b58442 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7a0bf52f4e7d466db4915af3c5b58442 2024-11-11T23:53:44,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7a0bf52f4e7d466db4915af3c5b58442, entries=200, sequenceid=461, filesize=14.4 K 2024-11-11T23:53:44,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/39153066e2994972be2d278fc9407f14 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/39153066e2994972be2d278fc9407f14 2024-11-11T23:53:44,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/39153066e2994972be2d278fc9407f14, entries=150, sequenceid=461, filesize=12.0 K 2024-11-11T23:53:44,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/f22a8a760e0e4a4b9fa9208a9683ac82 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f22a8a760e0e4a4b9fa9208a9683ac82 2024-11-11T23:53:44,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f22a8a760e0e4a4b9fa9208a9683ac82, entries=150, sequenceid=461, filesize=12.0 K 2024-11-11T23:53:44,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6b2061b8554b1e7422f6c037ae02cccd in 1277ms, sequenceid=461, compaction requested=true 2024-11-11T23:53:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:44,375 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:53:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:44,375 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:53:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:44,376 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:53:44,376 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:53:44,376 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:44,376 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:44,376 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:44,376 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:44,376 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8fc664f3cf9f41f8a7a3b03979aaf504, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b682d8f9bf554ba88b78dd51a436e363, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/edc6db057659442c85d64b1e401b32d5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/39153066e2994972be2d278fc9407f14] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=48.9 K 2024-11-11T23:53:44,376 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4b63fdad760a4f219d79d2483d0e3f9b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f8336b914daa4a20a4cae169f6bec3bb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7d2528726ec3436b90f67b5785c02985, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7a0bf52f4e7d466db4915af3c5b58442] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=51.3 K 2024-11-11T23:53:44,377 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc664f3cf9f41f8a7a3b03979aaf504, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369219566 2024-11-11T23:53:44,377 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b63fdad760a4f219d79d2483d0e3f9b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369219566 2024-11-11T23:53:44,377 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b682d8f9bf554ba88b78dd51a436e363, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731369220878 2024-11-11T23:53:44,377 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8336b914daa4a20a4cae169f6bec3bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731369220878 2024-11-11T23:53:44,377 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting edc6db057659442c85d64b1e401b32d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731369220911 2024-11-11T23:53:44,377 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d2528726ec3436b90f67b5785c02985, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731369220911 2024-11-11T23:53:44,378 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 39153066e2994972be2d278fc9407f14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731369222014 2024-11-11T23:53:44,378 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0bf52f4e7d466db4915af3c5b58442, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731369222014 2024-11-11T23:53:44,388 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:44,389 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/46a66cd43db446b385c3587b47244f82 is 50, key is test_row_0/A:col10/1731369222025/Put/seqid=0 2024-11-11T23:53:44,392 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:44,393 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/858ecfd8973744999197804c50b1a967 is 50, key is test_row_0/B:col10/1731369222025/Put/seqid=0 2024-11-11T23:53:44,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741924_1100 (size=13289) 2024-11-11T23:53:44,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741925_1101 (size=13289) 2024-11-11T23:53:44,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T23:53:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:44,406 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T23:53:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:44,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/b9c4aa8b3ea846cba028ba8ae3ccedc7 is 50, key is test_row_0/A:col10/1731369223123/Put/seqid=0 2024-11-11T23:53:44,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741926_1102 (size=12301) 2024-11-11T23:53:44,819 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/b9c4aa8b3ea846cba028ba8ae3ccedc7 2024-11-11T23:53:44,819 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/46a66cd43db446b385c3587b47244f82 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/46a66cd43db446b385c3587b47244f82 2024-11-11T23:53:44,821 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/858ecfd8973744999197804c50b1a967 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/858ecfd8973744999197804c50b1a967 2024-11-11T23:53:44,828 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into 46a66cd43db446b385c3587b47244f82(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:44,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:44,828 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=12, startTime=1731369224374; duration=0sec 2024-11-11T23:53:44,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:44,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:44,829 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:53:44,831 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:53:44,831 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:44,832 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:44,832 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/6cc100116036467ebbec9916c17a9922, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0076ef879f2c483e960636c03d381777, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2e39cfd60ae6446a9ab983c4f038d6a3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f22a8a760e0e4a4b9fa9208a9683ac82] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=48.9 K 2024-11-11T23:53:44,833 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cc100116036467ebbec9916c17a9922, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369219566 2024-11-11T23:53:44,834 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0076ef879f2c483e960636c03d381777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731369220878 2024-11-11T23:53:44,835 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into 858ecfd8973744999197804c50b1a967(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:44,835 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:44,835 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=12, startTime=1731369224375; duration=0sec 2024-11-11T23:53:44,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/700dcd147001468a9bd64e50d5a8d947 is 50, key is test_row_0/B:col10/1731369223123/Put/seqid=0 2024-11-11T23:53:44,836 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:44,836 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:44,836 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e39cfd60ae6446a9ab983c4f038d6a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731369220911 2024-11-11T23:53:44,840 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22a8a760e0e4a4b9fa9208a9683ac82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731369222014 2024-11-11T23:53:44,859 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:44,860 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/80ac6c4d380e4f39ae12d4b6aa2062d1 is 50, key is test_row_0/C:col10/1731369222025/Put/seqid=0 2024-11-11T23:53:44,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741928_1104 (size=13289) 2024-11-11T23:53:44,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741927_1103 (size=12301) 2024-11-11T23:53:44,875 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/700dcd147001468a9bd64e50d5a8d947 2024-11-11T23:53:44,881 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/80ac6c4d380e4f39ae12d4b6aa2062d1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/80ac6c4d380e4f39ae12d4b6aa2062d1 2024-11-11T23:53:44,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/8d438dffe4834b66b7549ea3929d7623 is 50, key is test_row_0/C:col10/1731369223123/Put/seqid=0 2024-11-11T23:53:44,890 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into 80ac6c4d380e4f39ae12d4b6aa2062d1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:44,890 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:44,890 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=12, startTime=1731369224375; duration=0sec 2024-11-11T23:53:44,890 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:44,890 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:44,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741929_1105 (size=12301) 2024-11-11T23:53:45,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. as already flushing 2024-11-11T23:53:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T23:53:45,329 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/8d438dffe4834b66b7549ea3929d7623 2024-11-11T23:53:45,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369285332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369285334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:45,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/b9c4aa8b3ea846cba028ba8ae3ccedc7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/b9c4aa8b3ea846cba028ba8ae3ccedc7 2024-11-11T23:53:45,344 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/b9c4aa8b3ea846cba028ba8ae3ccedc7, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T23:53:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/700dcd147001468a9bd64e50d5a8d947 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/700dcd147001468a9bd64e50d5a8d947 2024-11-11T23:53:45,351 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/700dcd147001468a9bd64e50d5a8d947, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T23:53:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/8d438dffe4834b66b7549ea3929d7623 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/8d438dffe4834b66b7549ea3929d7623 2024-11-11T23:53:45,364 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/8d438dffe4834b66b7549ea3929d7623, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T23:53:45,366 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6b2061b8554b1e7422f6c037ae02cccd in 959ms, sequenceid=473, compaction requested=false 2024-11-11T23:53:45,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:45,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:45,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T23:53:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T23:53:45,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T23:53:45,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1900 sec 2024-11-11T23:53:45,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.1960 sec 2024-11-11T23:53:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:45,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T23:53:45,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369225323, max timestamp=1731369225445Num uniques 859; 2024-11-11T23:53:45,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369225323, max timestamp=1731369225445Num uniques 859; 2024-11-11T23:53:45,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:45,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:45,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:45,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:45,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:45,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:45,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:45,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:45,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6b817ca0e51a4771bb8f7f359b726116 is 50, key is test_row_0/A:col10/1731369225445/Put/seqid=0 2024-11-11T23:53:45,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741930_1106 (size=14741) 2024-11-11T23:53:45,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6b817ca0e51a4771bb8f7f359b726116 2024-11-11T23:53:45,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a0bfc4477b3344fd92fa37d7e9ef861f is 50, key is test_row_0/B:col10/1731369225445/Put/seqid=0 2024-11-11T23:53:45,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741931_1107 (size=12301) 2024-11-11T23:53:45,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369285516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:45,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:45,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369285516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:45,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:45,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369285624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:45,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369285624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:45,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369285835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369285835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:45,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a0bfc4477b3344fd92fa37d7e9ef861f 2024-11-11T23:53:45,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/0c355136003541709378c5a22b80ada8 is 50, key is test_row_0/C:col10/1731369225445/Put/seqid=0 2024-11-11T23:53:45,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741932_1108 (size=12301) 2024-11-11T23:53:45,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/0c355136003541709378c5a22b80ada8 2024-11-11T23:53:45,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6b817ca0e51a4771bb8f7f359b726116 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b817ca0e51a4771bb8f7f359b726116 2024-11-11T23:53:45,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b817ca0e51a4771bb8f7f359b726116, entries=200, sequenceid=498, filesize=14.4 K 2024-11-11T23:53:45,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a0bfc4477b3344fd92fa37d7e9ef861f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a0bfc4477b3344fd92fa37d7e9ef861f 2024-11-11T23:53:45,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a0bfc4477b3344fd92fa37d7e9ef861f, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T23:53:45,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/0c355136003541709378c5a22b80ada8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0c355136003541709378c5a22b80ada8 2024-11-11T23:53:45,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0c355136003541709378c5a22b80ada8, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T23:53:45,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 6b2061b8554b1e7422f6c037ae02cccd in 543ms, sequenceid=498, compaction requested=true 2024-11-11T23:53:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:45,990 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:45,990 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:45,992 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:45,992 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:45,992 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:45,992 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:45,992 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:45,992 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:45,992 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/46a66cd43db446b385c3587b47244f82, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/b9c4aa8b3ea846cba028ba8ae3ccedc7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b817ca0e51a4771bb8f7f359b726116] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=39.4 K 2024-11-11T23:53:45,992 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/858ecfd8973744999197804c50b1a967, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/700dcd147001468a9bd64e50d5a8d947, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a0bfc4477b3344fd92fa37d7e9ef861f] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=37.0 K 2024-11-11T23:53:45,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:45,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:45,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:45,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:45,994 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 858ecfd8973744999197804c50b1a967, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731369222014 2024-11-11T23:53:45,995 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46a66cd43db446b385c3587b47244f82, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731369222014 2024-11-11T23:53:45,995 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 700dcd147001468a9bd64e50d5a8d947, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731369223110 2024-11-11T23:53:45,996 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c4aa8b3ea846cba028ba8ae3ccedc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731369223110 2024-11-11T23:53:45,996 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a0bfc4477b3344fd92fa37d7e9ef861f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369225331 2024-11-11T23:53:45,996 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b817ca0e51a4771bb8f7f359b726116, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369225329 2024-11-11T23:53:46,020 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:46,020 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:46,021 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/ebde83ee56c8411da0b22fe2e645be47 is 50, key is test_row_0/B:col10/1731369225445/Put/seqid=0 2024-11-11T23:53:46,021 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/f083d9b6570a40d69d89316cf065f2c4 is 50, key is test_row_0/A:col10/1731369225445/Put/seqid=0 2024-11-11T23:53:46,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741933_1109 (size=13391) 2024-11-11T23:53:46,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741934_1110 (size=13391) 2024-11-11T23:53:46,080 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/f083d9b6570a40d69d89316cf065f2c4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f083d9b6570a40d69d89316cf065f2c4 2024-11-11T23:53:46,091 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/ebde83ee56c8411da0b22fe2e645be47 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ebde83ee56c8411da0b22fe2e645be47 2024-11-11T23:53:46,102 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into ebde83ee56c8411da0b22fe2e645be47(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:46,102 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:46,102 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369225990; duration=0sec 2024-11-11T23:53:46,102 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into f083d9b6570a40d69d89316cf065f2c4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:46,102 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:46,102 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:46,102 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369225990; duration=0sec 2024-11-11T23:53:46,102 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:46,102 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:46,102 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:46,102 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:46,104 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:46,104 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:46,104 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:46,104 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/80ac6c4d380e4f39ae12d4b6aa2062d1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/8d438dffe4834b66b7549ea3929d7623, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0c355136003541709378c5a22b80ada8] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=37.0 K 2024-11-11T23:53:46,105 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ac6c4d380e4f39ae12d4b6aa2062d1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731369222014 2024-11-11T23:53:46,106 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d438dffe4834b66b7549ea3929d7623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731369223110 2024-11-11T23:53:46,106 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c355136003541709378c5a22b80ada8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369225331 2024-11-11T23:53:46,119 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:46,120 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b9da267cf95b4ff6b8d33dd6fc427496 is 50, key is test_row_0/C:col10/1731369225445/Put/seqid=0 2024-11-11T23:53:46,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741935_1111 (size=13391) 2024-11-11T23:53:46,147 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b9da267cf95b4ff6b8d33dd6fc427496 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b9da267cf95b4ff6b8d33dd6fc427496 2024-11-11T23:53:46,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369225488, max timestamp=1731369226148Num uniques 859; 2024-11-11T23:53:46,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369225488, max timestamp=1731369226148Num uniques 859; 2024-11-11T23:53:46,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:46,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:46,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369225488, max timestamp=1731369226148Num uniques 859; 2024-11-11T23:53:46,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:46,158 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into b9da267cf95b4ff6b8d33dd6fc427496(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:46,158 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:46,158 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369225993; duration=0sec 2024-11-11T23:53:46,159 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:46,159 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:46,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-11T23:53:46,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:46,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:46,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:46,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:46,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:46,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:46,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/4d236679dbd84adf9dd4ff8de97b7f17 is 50, key is test_row_0/A:col10/1731369226166/Put/seqid=0 2024-11-11T23:53:46,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741936_1112 (size=17181) 2024-11-11T23:53:46,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369286218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369226209, max timestamp=1731369226228Num uniques 859; 2024-11-11T23:53:46,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369226209, max timestamp=1731369226228Num uniques 859; 2024-11-11T23:53:46,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:46,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369226209, max timestamp=1731369226228Num uniques 859; 2024-11-11T23:53:46,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:53:46,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369286294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369286294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369286296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369286344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369286404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369286416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369286416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369286554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/4d236679dbd84adf9dd4ff8de97b7f17 2024-11-11T23:53:46,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/d89c351c389a42fd820bedaf99147845 is 50, key is test_row_0/B:col10/1731369226166/Put/seqid=0 2024-11-11T23:53:46,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369286624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369286635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369286635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741937_1113 (size=12301) 2024-11-11T23:53:46,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/d89c351c389a42fd820bedaf99147845 2024-11-11T23:53:46,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b8aa787b89a949f4a26c407765dd3529 is 50, key is test_row_0/C:col10/1731369226166/Put/seqid=0 2024-11-11T23:53:46,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741938_1114 (size=12301) 2024-11-11T23:53:46,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35716 deadline: 1731369286865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35722 deadline: 1731369286944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35688 deadline: 1731369286944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:46,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35678 deadline: 1731369286955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:53:47,088 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T23:53:47,088 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:47,094 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T23:53:47,094 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:47,096 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,096 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,097 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:47,098 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:47,099 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T23:53:47,099 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:47,099 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,099 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:47,103 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T23:53:47,103 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:47,103 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,104 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:47,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b8aa787b89a949f4a26c407765dd3529 2024-11-11T23:53:47,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/4d236679dbd84adf9dd4ff8de97b7f17 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4d236679dbd84adf9dd4ff8de97b7f17 2024-11-11T23:53:47,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4d236679dbd84adf9dd4ff8de97b7f17, entries=250, sequenceid=527, filesize=16.8 K 2024-11-11T23:53:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/d89c351c389a42fd820bedaf99147845 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/d89c351c389a42fd820bedaf99147845 2024-11-11T23:53:47,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/d89c351c389a42fd820bedaf99147845, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T23:53:47,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/b8aa787b89a949f4a26c407765dd3529 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b8aa787b89a949f4a26c407765dd3529 2024-11-11T23:53:47,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b8aa787b89a949f4a26c407765dd3529, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T23:53:47,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for 6b2061b8554b1e7422f6c037ae02cccd in 989ms, sequenceid=527, compaction requested=false 2024-11-11T23:53:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T23:53:47,345 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:47,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T23:53:47,387 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T23:53:47,387 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:47,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:47,387 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:47,388 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:47,388 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:47,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6e27e334554545ae8bb8c5a3fcd59e4e is 50, key is test_row_0/A:col10/1731369227385/Put/seqid=0 2024-11-11T23:53:47,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741939_1115 (size=12301) 2024-11-11T23:53:47,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6e27e334554545ae8bb8c5a3fcd59e4e 2024-11-11T23:53:47,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a34c20279bae4c40899785d2fd415427 is 50, key is test_row_0/B:col10/1731369227385/Put/seqid=0 2024-11-11T23:53:47,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741940_1116 (size=12301) 2024-11-11T23:53:47,456 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T23:53:47,456 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:47,457 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,457 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:47,466 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T23:53:47,466 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T23:53:47,466 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:47,466 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:47,466 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,466 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,466 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,466 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:47,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:47,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:47,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a34c20279bae4c40899785d2fd415427 2024-11-11T23:53:47,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/455a74c1658f458c9e43cd0a2c2e01e2 is 50, key is test_row_0/C:col10/1731369227385/Put/seqid=0 2024-11-11T23:53:47,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741941_1117 (size=12301) 2024-11-11T23:53:48,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/455a74c1658f458c9e43cd0a2c2e01e2 2024-11-11T23:53:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/6e27e334554545ae8bb8c5a3fcd59e4e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6e27e334554545ae8bb8c5a3fcd59e4e 2024-11-11T23:53:48,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6e27e334554545ae8bb8c5a3fcd59e4e, entries=150, sequenceid=563, filesize=12.0 K 2024-11-11T23:53:48,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/a34c20279bae4c40899785d2fd415427 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a34c20279bae4c40899785d2fd415427 2024-11-11T23:53:48,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a34c20279bae4c40899785d2fd415427, entries=150, sequenceid=563, filesize=12.0 K 2024-11-11T23:53:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/455a74c1658f458c9e43cd0a2c2e01e2 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/455a74c1658f458c9e43cd0a2c2e01e2 2024-11-11T23:53:48,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/455a74c1658f458c9e43cd0a2c2e01e2, entries=150, sequenceid=563, filesize=12.0 K 2024-11-11T23:53:48,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=20.13 KB/20610 for 6b2061b8554b1e7422f6c037ae02cccd in 878ms, sequenceid=563, compaction requested=true 2024-11-11T23:53:48,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:48,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:53:48,265 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:48,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:48,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:53:48,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:48,266 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:48,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b2061b8554b1e7422f6c037ae02cccd:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:53:48,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:48,267 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:48,267 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/A is initiating minor compaction (all files) 2024-11-11T23:53:48,267 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/A in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:48,267 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f083d9b6570a40d69d89316cf065f2c4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4d236679dbd84adf9dd4ff8de97b7f17, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6e27e334554545ae8bb8c5a3fcd59e4e] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=41.9 K 2024-11-11T23:53:48,267 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:48,267 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/B is initiating minor compaction (all files) 2024-11-11T23:53:48,267 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f083d9b6570a40d69d89316cf065f2c4, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369225331 2024-11-11T23:53:48,267 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/B in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:48,268 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ebde83ee56c8411da0b22fe2e645be47, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/d89c351c389a42fd820bedaf99147845, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a34c20279bae4c40899785d2fd415427] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=37.1 K 2024-11-11T23:53:48,268 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d236679dbd84adf9dd4ff8de97b7f17, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731369226148 2024-11-11T23:53:48,268 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ebde83ee56c8411da0b22fe2e645be47, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369225331 2024-11-11T23:53:48,268 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e27e334554545ae8bb8c5a3fcd59e4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731369226260 2024-11-11T23:53:48,268 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting d89c351c389a42fd820bedaf99147845, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731369226163 2024-11-11T23:53:48,268 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a34c20279bae4c40899785d2fd415427, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731369226260 2024-11-11T23:53:48,293 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#B#compaction#105 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:48,293 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/28b45fced56545afb16814e91a1f83ee is 50, key is test_row_0/B:col10/1731369227385/Put/seqid=0 2024-11-11T23:53:48,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741942_1118 (size=13493) 2024-11-11T23:53:48,308 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#A#compaction#106 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:48,309 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/a4288308c73b4260982c5d91135167a3 is 50, key is test_row_0/A:col10/1731369227385/Put/seqid=0 2024-11-11T23:53:48,311 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/28b45fced56545afb16814e91a1f83ee as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/28b45fced56545afb16814e91a1f83ee 2024-11-11T23:53:48,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741943_1119 (size=13493) 2024-11-11T23:53:48,319 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/B of 6b2061b8554b1e7422f6c037ae02cccd into 28b45fced56545afb16814e91a1f83ee(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:48,319 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/a4288308c73b4260982c5d91135167a3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a4288308c73b4260982c5d91135167a3 2024-11-11T23:53:48,319 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:48,319 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/B, priority=13, startTime=1731369228265; duration=0sec 2024-11-11T23:53:48,319 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:53:48,319 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:B 2024-11-11T23:53:48,319 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:53:48,320 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:53:48,320 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6b2061b8554b1e7422f6c037ae02cccd/C is initiating minor compaction (all files) 2024-11-11T23:53:48,321 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b2061b8554b1e7422f6c037ae02cccd/C in TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:48,321 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b9da267cf95b4ff6b8d33dd6fc427496, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b8aa787b89a949f4a26c407765dd3529, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/455a74c1658f458c9e43cd0a2c2e01e2] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp, totalSize=37.1 K 2024-11-11T23:53:48,321 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b9da267cf95b4ff6b8d33dd6fc427496, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369225331 2024-11-11T23:53:48,321 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b8aa787b89a949f4a26c407765dd3529, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731369226163 2024-11-11T23:53:48,322 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 455a74c1658f458c9e43cd0a2c2e01e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731369226260 2024-11-11T23:53:48,325 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/A of 6b2061b8554b1e7422f6c037ae02cccd into a4288308c73b4260982c5d91135167a3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:48,325 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:48,325 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/A, priority=13, startTime=1731369228265; duration=0sec 2024-11-11T23:53:48,325 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:48,325 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:A 2024-11-11T23:53:48,334 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b2061b8554b1e7422f6c037ae02cccd#C#compaction#107 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:53:48,335 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/e6ff7c2071f14f0488f0722a378d63e7 is 50, key is test_row_0/C:col10/1731369227385/Put/seqid=0 2024-11-11T23:53:48,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741944_1120 (size=13493) 2024-11-11T23:53:48,746 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/e6ff7c2071f14f0488f0722a378d63e7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/e6ff7c2071f14f0488f0722a378d63e7 2024-11-11T23:53:48,751 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b2061b8554b1e7422f6c037ae02cccd/C of 6b2061b8554b1e7422f6c037ae02cccd into e6ff7c2071f14f0488f0722a378d63e7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:53:48,751 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b2061b8554b1e7422f6c037ae02cccd: 2024-11-11T23:53:48,751 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., storeName=6b2061b8554b1e7422f6c037ae02cccd/C, priority=13, startTime=1731369228266; duration=0sec 2024-11-11T23:53:48,752 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:53:48,752 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b2061b8554b1e7422f6c037ae02cccd:C 2024-11-11T23:53:49,730 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:53:51,077 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T23:53:51,077 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:53:51,077 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:51,077 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:51,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:51,077 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2651 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2669 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1187 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3561 rows 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1196 2024-11-11T23:53:51,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3587 rows 2024-11-11T23:53:51,078 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:53:51,079 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:53:51,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:51,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:51,079 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:53:51,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T23:53:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T23:53:51,094 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369231094"}]},"ts":"1731369231094"} 2024-11-11T23:53:51,098 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T23:53:51,098 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T23:53:51,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:53:51,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, UNASSIGN}] 2024-11-11T23:53:51,108 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, UNASSIGN 2024-11-11T23:53:51,110 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6b2061b8554b1e7422f6c037ae02cccd, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:51,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, UNASSIGN because future has completed 2024-11-11T23:53:51,113 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:53:51,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:53:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T23:53:51,270 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:51,271 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:53:51,271 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 6b2061b8554b1e7422f6c037ae02cccd, disabling compactions & flushes 2024-11-11T23:53:51,271 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. after waiting 0 ms 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:51,272 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 6b2061b8554b1e7422f6c037ae02cccd 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=A 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=B 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6b2061b8554b1e7422f6c037ae02cccd, store=C 2024-11-11T23:53:51,272 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:51,278 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/2a53d7ee735743f28728b5d6c6c937f3 is 50, key is test_row_0/A:col10/1731369227465/Put/seqid=0 2024-11-11T23:53:51,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741945_1121 (size=9857) 2024-11-11T23:53:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T23:53:51,684 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/2a53d7ee735743f28728b5d6c6c937f3 2024-11-11T23:53:51,695 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/cc72c9f61f914e68a0062259bb0318b8 is 50, key is test_row_0/B:col10/1731369227465/Put/seqid=0 2024-11-11T23:53:51,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741946_1122 (size=9857) 2024-11-11T23:53:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T23:53:52,101 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/cc72c9f61f914e68a0062259bb0318b8 2024-11-11T23:53:52,117 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/5982b2f081b54ce7b17e4ce9cccab757 is 50, key is test_row_0/C:col10/1731369227465/Put/seqid=0 2024-11-11T23:53:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741947_1123 (size=9857) 2024-11-11T23:53:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T23:53:52,523 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/5982b2f081b54ce7b17e4ce9cccab757 2024-11-11T23:53:52,534 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/A/2a53d7ee735743f28728b5d6c6c937f3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/2a53d7ee735743f28728b5d6c6c937f3 2024-11-11T23:53:52,538 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/2a53d7ee735743f28728b5d6c6c937f3, entries=100, sequenceid=573, filesize=9.6 K 2024-11-11T23:53:52,539 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/B/cc72c9f61f914e68a0062259bb0318b8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/cc72c9f61f914e68a0062259bb0318b8 2024-11-11T23:53:52,544 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/cc72c9f61f914e68a0062259bb0318b8, entries=100, sequenceid=573, filesize=9.6 K 2024-11-11T23:53:52,545 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/.tmp/C/5982b2f081b54ce7b17e4ce9cccab757 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/5982b2f081b54ce7b17e4ce9cccab757 2024-11-11T23:53:52,549 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/5982b2f081b54ce7b17e4ce9cccab757, entries=100, sequenceid=573, filesize=9.6 K 2024-11-11T23:53:52,550 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6b2061b8554b1e7422f6c037ae02cccd in 1278ms, sequenceid=573, compaction requested=false 2024-11-11T23:53:52,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a38e230ad065473582b53f8677ac01df, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/efd4a0ee650e49ba8b8e58d4db18d5ac, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/89d6662bc63441e1bb5bc153ba6921f0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7f79bba16c804ebe8b34177589be14ae, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/bab34a8d1c5a4be7a582b129f197c5c5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/69e449cdc4db4564ba03afa1023aa0e3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/573d948c57124e3cb52511d955ab6e90, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/cc2be082a02b4477a7614ac49b42886d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/38df981668234d18bd0389650b2cc002, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6c1f5ac330424205a79134beec7ea46d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/11cba5d8e6ed4737bd8a3d636abc82cd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/fdc420a835a3471da425b8c34f4e8b3f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/43807b914896481b917c6bcc1c4d09e0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6f03ba982f7240bfb00a5241578f09e1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a5a510320d6f4cb3a630af7ffc0b06d2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b34901033204271a219cf22f16e3fe1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/3086d9aa63134785b670be74384e86b9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/34ffb723bf68418da04dc5b5e7c59578, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/d32abe9da26f49ba8f6e8420fa57859c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7916a5449ccb4abea2157dab410990e5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/263f68605a684a79ac5b6970cfc8ae55, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/de0e1f0d15fd4aeb9963e669a5675b79, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/49a8a0af023d46cfa4cbe789dcd4db0a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/020d267db3154818bc769ebc527a07b6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/492c8987e559430797ddc49bebd92c70, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4b63fdad760a4f219d79d2483d0e3f9b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f8336b914daa4a20a4cae169f6bec3bb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7d2528726ec3436b90f67b5785c02985, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7a0bf52f4e7d466db4915af3c5b58442, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/46a66cd43db446b385c3587b47244f82, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/b9c4aa8b3ea846cba028ba8ae3ccedc7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b817ca0e51a4771bb8f7f359b726116, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f083d9b6570a40d69d89316cf065f2c4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4d236679dbd84adf9dd4ff8de97b7f17, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6e27e334554545ae8bb8c5a3fcd59e4e] to archive 2024-11-11T23:53:52,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:53:52,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a38e230ad065473582b53f8677ac01df to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a38e230ad065473582b53f8677ac01df 2024-11-11T23:53:52,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/efd4a0ee650e49ba8b8e58d4db18d5ac to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/efd4a0ee650e49ba8b8e58d4db18d5ac 2024-11-11T23:53:52,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/89d6662bc63441e1bb5bc153ba6921f0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/89d6662bc63441e1bb5bc153ba6921f0 2024-11-11T23:53:52,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7f79bba16c804ebe8b34177589be14ae to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7f79bba16c804ebe8b34177589be14ae 2024-11-11T23:53:52,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/bab34a8d1c5a4be7a582b129f197c5c5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/bab34a8d1c5a4be7a582b129f197c5c5 2024-11-11T23:53:52,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/69e449cdc4db4564ba03afa1023aa0e3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/69e449cdc4db4564ba03afa1023aa0e3 2024-11-11T23:53:52,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/573d948c57124e3cb52511d955ab6e90 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/573d948c57124e3cb52511d955ab6e90 2024-11-11T23:53:52,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/cc2be082a02b4477a7614ac49b42886d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/cc2be082a02b4477a7614ac49b42886d 2024-11-11T23:53:52,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/38df981668234d18bd0389650b2cc002 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/38df981668234d18bd0389650b2cc002 2024-11-11T23:53:52,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6c1f5ac330424205a79134beec7ea46d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6c1f5ac330424205a79134beec7ea46d 2024-11-11T23:53:52,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/11cba5d8e6ed4737bd8a3d636abc82cd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/11cba5d8e6ed4737bd8a3d636abc82cd 2024-11-11T23:53:52,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/fdc420a835a3471da425b8c34f4e8b3f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/fdc420a835a3471da425b8c34f4e8b3f 2024-11-11T23:53:52,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/43807b914896481b917c6bcc1c4d09e0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/43807b914896481b917c6bcc1c4d09e0 2024-11-11T23:53:52,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6f03ba982f7240bfb00a5241578f09e1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6f03ba982f7240bfb00a5241578f09e1 2024-11-11T23:53:52,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a5a510320d6f4cb3a630af7ffc0b06d2 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a5a510320d6f4cb3a630af7ffc0b06d2 2024-11-11T23:53:52,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b34901033204271a219cf22f16e3fe1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b34901033204271a219cf22f16e3fe1 2024-11-11T23:53:52,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/3086d9aa63134785b670be74384e86b9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/3086d9aa63134785b670be74384e86b9 2024-11-11T23:53:52,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/34ffb723bf68418da04dc5b5e7c59578 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/34ffb723bf68418da04dc5b5e7c59578 2024-11-11T23:53:52,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/d32abe9da26f49ba8f6e8420fa57859c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/d32abe9da26f49ba8f6e8420fa57859c 2024-11-11T23:53:52,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7916a5449ccb4abea2157dab410990e5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7916a5449ccb4abea2157dab410990e5 2024-11-11T23:53:52,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/263f68605a684a79ac5b6970cfc8ae55 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/263f68605a684a79ac5b6970cfc8ae55 2024-11-11T23:53:52,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/de0e1f0d15fd4aeb9963e669a5675b79 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/de0e1f0d15fd4aeb9963e669a5675b79 2024-11-11T23:53:52,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/49a8a0af023d46cfa4cbe789dcd4db0a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/49a8a0af023d46cfa4cbe789dcd4db0a 2024-11-11T23:53:52,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/020d267db3154818bc769ebc527a07b6 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/020d267db3154818bc769ebc527a07b6 2024-11-11T23:53:52,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/492c8987e559430797ddc49bebd92c70 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/492c8987e559430797ddc49bebd92c70 2024-11-11T23:53:52,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4b63fdad760a4f219d79d2483d0e3f9b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4b63fdad760a4f219d79d2483d0e3f9b 2024-11-11T23:53:52,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f8336b914daa4a20a4cae169f6bec3bb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f8336b914daa4a20a4cae169f6bec3bb 2024-11-11T23:53:52,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7d2528726ec3436b90f67b5785c02985 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7d2528726ec3436b90f67b5785c02985 2024-11-11T23:53:52,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7a0bf52f4e7d466db4915af3c5b58442 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/7a0bf52f4e7d466db4915af3c5b58442 2024-11-11T23:53:52,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/46a66cd43db446b385c3587b47244f82 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/46a66cd43db446b385c3587b47244f82 2024-11-11T23:53:52,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/b9c4aa8b3ea846cba028ba8ae3ccedc7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/b9c4aa8b3ea846cba028ba8ae3ccedc7 2024-11-11T23:53:52,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b817ca0e51a4771bb8f7f359b726116 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6b817ca0e51a4771bb8f7f359b726116 2024-11-11T23:53:52,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f083d9b6570a40d69d89316cf065f2c4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/f083d9b6570a40d69d89316cf065f2c4 2024-11-11T23:53:52,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4d236679dbd84adf9dd4ff8de97b7f17 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/4d236679dbd84adf9dd4ff8de97b7f17 2024-11-11T23:53:52,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6e27e334554545ae8bb8c5a3fcd59e4e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/6e27e334554545ae8bb8c5a3fcd59e4e 2024-11-11T23:53:52,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b8ee74b0187c493ca8e213e44d261bb4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c4307fe9a3f74f979ef2fbc2acdbbdf7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8b3f84b5ca174282808eea81f337a332, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/98ac11d93ec740c7a725509ee22ef1db, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f9ebbaea87074ed4a5e9bab74c4b512b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2107f983dc4b46b390458390f64ca911, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/37e8e410aeee4ca0b4e82b0b99645798, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1dbe0334ab884ec0a7a767e5a0ce3abb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/dc508db133f446c6a389748e08a8738c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/aafbb821305747f9a2b46dda4073b90c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1fef74c0c1fc4cd6b6602151cf9ac2d9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ab466e8dc4e34ae7aed145c2c53b9c60, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5b1eedd67850433b9055442a2d742bb9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5930e33ecfca4354a873d70bed91458b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2b4049a99ad84e9588ccb5db3d6bae0a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ad7fcc1fae6f49bb90998e5780a20b8c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a1f7e907d3aa4803a284a13b919921c0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4e1deb74ec834d22a72e5de784651664, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f6bff28adf5847f6aaa087cdcaf92bf6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4f8837680ea14e26ab9911fe758e42ac, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/302c299ffdfa4fbba7e7c259092b2de2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c86e6892385d46f6a7fbaf955593a5e5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/64018c4ec52846dc8482c756da0660a8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/3dbc7c8e01c8479299bf3239bd150bc1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8fc664f3cf9f41f8a7a3b03979aaf504, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/32332ee01f32437bac14364d2fd6546f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b682d8f9bf554ba88b78dd51a436e363, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/edc6db057659442c85d64b1e401b32d5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/858ecfd8973744999197804c50b1a967, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/39153066e2994972be2d278fc9407f14, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/700dcd147001468a9bd64e50d5a8d947, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ebde83ee56c8411da0b22fe2e645be47, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a0bfc4477b3344fd92fa37d7e9ef861f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/d89c351c389a42fd820bedaf99147845, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a34c20279bae4c40899785d2fd415427] to archive 2024-11-11T23:53:52,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:53:52,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b8ee74b0187c493ca8e213e44d261bb4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b8ee74b0187c493ca8e213e44d261bb4 2024-11-11T23:53:52,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c4307fe9a3f74f979ef2fbc2acdbbdf7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c4307fe9a3f74f979ef2fbc2acdbbdf7 2024-11-11T23:53:52,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8b3f84b5ca174282808eea81f337a332 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8b3f84b5ca174282808eea81f337a332 2024-11-11T23:53:52,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/98ac11d93ec740c7a725509ee22ef1db to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/98ac11d93ec740c7a725509ee22ef1db 2024-11-11T23:53:52,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f9ebbaea87074ed4a5e9bab74c4b512b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f9ebbaea87074ed4a5e9bab74c4b512b 2024-11-11T23:53:52,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2107f983dc4b46b390458390f64ca911 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2107f983dc4b46b390458390f64ca911 2024-11-11T23:53:52,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/37e8e410aeee4ca0b4e82b0b99645798 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/37e8e410aeee4ca0b4e82b0b99645798 2024-11-11T23:53:52,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1dbe0334ab884ec0a7a767e5a0ce3abb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1dbe0334ab884ec0a7a767e5a0ce3abb 2024-11-11T23:53:52,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/dc508db133f446c6a389748e08a8738c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/dc508db133f446c6a389748e08a8738c 2024-11-11T23:53:52,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/aafbb821305747f9a2b46dda4073b90c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/aafbb821305747f9a2b46dda4073b90c 2024-11-11T23:53:52,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1fef74c0c1fc4cd6b6602151cf9ac2d9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/1fef74c0c1fc4cd6b6602151cf9ac2d9 2024-11-11T23:53:52,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ab466e8dc4e34ae7aed145c2c53b9c60 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ab466e8dc4e34ae7aed145c2c53b9c60 2024-11-11T23:53:52,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5b1eedd67850433b9055442a2d742bb9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5b1eedd67850433b9055442a2d742bb9 2024-11-11T23:53:52,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5930e33ecfca4354a873d70bed91458b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/5930e33ecfca4354a873d70bed91458b 2024-11-11T23:53:52,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2b4049a99ad84e9588ccb5db3d6bae0a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/2b4049a99ad84e9588ccb5db3d6bae0a 2024-11-11T23:53:52,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ad7fcc1fae6f49bb90998e5780a20b8c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ad7fcc1fae6f49bb90998e5780a20b8c 2024-11-11T23:53:52,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a1f7e907d3aa4803a284a13b919921c0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a1f7e907d3aa4803a284a13b919921c0 2024-11-11T23:53:52,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4e1deb74ec834d22a72e5de784651664 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4e1deb74ec834d22a72e5de784651664 2024-11-11T23:53:52,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f6bff28adf5847f6aaa087cdcaf92bf6 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/f6bff28adf5847f6aaa087cdcaf92bf6 2024-11-11T23:53:52,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4f8837680ea14e26ab9911fe758e42ac to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/4f8837680ea14e26ab9911fe758e42ac 2024-11-11T23:53:52,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/302c299ffdfa4fbba7e7c259092b2de2 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/302c299ffdfa4fbba7e7c259092b2de2 2024-11-11T23:53:52,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c86e6892385d46f6a7fbaf955593a5e5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/c86e6892385d46f6a7fbaf955593a5e5 2024-11-11T23:53:52,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/64018c4ec52846dc8482c756da0660a8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/64018c4ec52846dc8482c756da0660a8 2024-11-11T23:53:52,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/3dbc7c8e01c8479299bf3239bd150bc1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/3dbc7c8e01c8479299bf3239bd150bc1 2024-11-11T23:53:52,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8fc664f3cf9f41f8a7a3b03979aaf504 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/8fc664f3cf9f41f8a7a3b03979aaf504 2024-11-11T23:53:52,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/32332ee01f32437bac14364d2fd6546f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/32332ee01f32437bac14364d2fd6546f 2024-11-11T23:53:52,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b682d8f9bf554ba88b78dd51a436e363 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/b682d8f9bf554ba88b78dd51a436e363 2024-11-11T23:53:52,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/edc6db057659442c85d64b1e401b32d5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/edc6db057659442c85d64b1e401b32d5 2024-11-11T23:53:52,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/858ecfd8973744999197804c50b1a967 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/858ecfd8973744999197804c50b1a967 2024-11-11T23:53:52,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/39153066e2994972be2d278fc9407f14 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/39153066e2994972be2d278fc9407f14 2024-11-11T23:53:52,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/700dcd147001468a9bd64e50d5a8d947 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/700dcd147001468a9bd64e50d5a8d947 2024-11-11T23:53:52,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ebde83ee56c8411da0b22fe2e645be47 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/ebde83ee56c8411da0b22fe2e645be47 2024-11-11T23:53:52,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a0bfc4477b3344fd92fa37d7e9ef861f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a0bfc4477b3344fd92fa37d7e9ef861f 2024-11-11T23:53:52,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/d89c351c389a42fd820bedaf99147845 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/d89c351c389a42fd820bedaf99147845 2024-11-11T23:53:52,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a34c20279bae4c40899785d2fd415427 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/a34c20279bae4c40899785d2fd415427 2024-11-11T23:53:52,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/a77175e401b04b17bee91f6546d4fb80, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/1243c91eb74841fb89f25a8829d51fcb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/003cb90556c144c2b0f6a28e6715d11e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/31d51b3aabc242989f0b7a0164f66c66, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/ac3010eacd4f4fc8bec75765593a88c9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f3344eb4a9f34fbb9188aed49aa0aec9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/9a553b6adb4e4171adcfe447a42ba22a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/7e4d731e7f694be58106e3363bad9625, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b08fa650b9a147cc8b03d97012352ff8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/01a3cccd39b544908cf48e9c4a0b73fa, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/67c65f3af81844cba2d65c5b82ba38fe, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/04b677710f504598ae5d798016dd161c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/59b8a591c63a41b5951995039b982465, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/307ec42f5e6b4f529603ef4df3246085, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2afddd48c36344ceb82e4f5a3cddaac3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/41b6e30cb2a143e0a82535273771e3e4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b20c8b3bf48d46da930ca464eeb6c74e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/fe09cdc9a530419bb0289f3af8c4bb9a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/74de01ab7d704b5e9b7af521d081190e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/3398ec2f6cea4f4084dd86863b966d8b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/86b1bda9db024e30a12a2d317fa830ad, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/dcfe811213c741f48514ca0775eea43a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/582703095e274533bdf71eca2686c7bc, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/6cc100116036467ebbec9916c17a9922, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/32cb142ab646458b912386bf3ede224a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0076ef879f2c483e960636c03d381777, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2e39cfd60ae6446a9ab983c4f038d6a3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/80ac6c4d380e4f39ae12d4b6aa2062d1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f22a8a760e0e4a4b9fa9208a9683ac82, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/8d438dffe4834b66b7549ea3929d7623, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b9da267cf95b4ff6b8d33dd6fc427496, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0c355136003541709378c5a22b80ada8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b8aa787b89a949f4a26c407765dd3529, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/455a74c1658f458c9e43cd0a2c2e01e2] to archive 2024-11-11T23:53:52,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:53:52,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/a77175e401b04b17bee91f6546d4fb80 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/a77175e401b04b17bee91f6546d4fb80 2024-11-11T23:53:52,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/1243c91eb74841fb89f25a8829d51fcb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/1243c91eb74841fb89f25a8829d51fcb 2024-11-11T23:53:52,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/003cb90556c144c2b0f6a28e6715d11e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/003cb90556c144c2b0f6a28e6715d11e 2024-11-11T23:53:52,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/31d51b3aabc242989f0b7a0164f66c66 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/31d51b3aabc242989f0b7a0164f66c66 2024-11-11T23:53:52,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/ac3010eacd4f4fc8bec75765593a88c9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/ac3010eacd4f4fc8bec75765593a88c9 2024-11-11T23:53:52,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f3344eb4a9f34fbb9188aed49aa0aec9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f3344eb4a9f34fbb9188aed49aa0aec9 2024-11-11T23:53:52,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/9a553b6adb4e4171adcfe447a42ba22a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/9a553b6adb4e4171adcfe447a42ba22a 2024-11-11T23:53:52,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/7e4d731e7f694be58106e3363bad9625 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/7e4d731e7f694be58106e3363bad9625 2024-11-11T23:53:52,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b08fa650b9a147cc8b03d97012352ff8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b08fa650b9a147cc8b03d97012352ff8 2024-11-11T23:53:52,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/01a3cccd39b544908cf48e9c4a0b73fa to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/01a3cccd39b544908cf48e9c4a0b73fa 2024-11-11T23:53:52,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/67c65f3af81844cba2d65c5b82ba38fe to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/67c65f3af81844cba2d65c5b82ba38fe 2024-11-11T23:53:52,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/04b677710f504598ae5d798016dd161c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/04b677710f504598ae5d798016dd161c 2024-11-11T23:53:52,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/59b8a591c63a41b5951995039b982465 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/59b8a591c63a41b5951995039b982465 2024-11-11T23:53:52,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/307ec42f5e6b4f529603ef4df3246085 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/307ec42f5e6b4f529603ef4df3246085 2024-11-11T23:53:52,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2afddd48c36344ceb82e4f5a3cddaac3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2afddd48c36344ceb82e4f5a3cddaac3 2024-11-11T23:53:52,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/41b6e30cb2a143e0a82535273771e3e4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/41b6e30cb2a143e0a82535273771e3e4 2024-11-11T23:53:52,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b20c8b3bf48d46da930ca464eeb6c74e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b20c8b3bf48d46da930ca464eeb6c74e 2024-11-11T23:53:52,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/fe09cdc9a530419bb0289f3af8c4bb9a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/fe09cdc9a530419bb0289f3af8c4bb9a 2024-11-11T23:53:52,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/74de01ab7d704b5e9b7af521d081190e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/74de01ab7d704b5e9b7af521d081190e 2024-11-11T23:53:52,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/3398ec2f6cea4f4084dd86863b966d8b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/3398ec2f6cea4f4084dd86863b966d8b 2024-11-11T23:53:52,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/86b1bda9db024e30a12a2d317fa830ad to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/86b1bda9db024e30a12a2d317fa830ad 2024-11-11T23:53:52,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/dcfe811213c741f48514ca0775eea43a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/dcfe811213c741f48514ca0775eea43a 2024-11-11T23:53:52,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/582703095e274533bdf71eca2686c7bc to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/582703095e274533bdf71eca2686c7bc 2024-11-11T23:53:52,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b2bac6bf4bbb4e6daa5b73c25ebaa10f 2024-11-11T23:53:52,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/6cc100116036467ebbec9916c17a9922 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/6cc100116036467ebbec9916c17a9922 2024-11-11T23:53:52,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/32cb142ab646458b912386bf3ede224a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/32cb142ab646458b912386bf3ede224a 2024-11-11T23:53:52,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0076ef879f2c483e960636c03d381777 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0076ef879f2c483e960636c03d381777 2024-11-11T23:53:52,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2e39cfd60ae6446a9ab983c4f038d6a3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/2e39cfd60ae6446a9ab983c4f038d6a3 2024-11-11T23:53:52,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/80ac6c4d380e4f39ae12d4b6aa2062d1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/80ac6c4d380e4f39ae12d4b6aa2062d1 2024-11-11T23:53:52,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f22a8a760e0e4a4b9fa9208a9683ac82 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/f22a8a760e0e4a4b9fa9208a9683ac82 2024-11-11T23:53:52,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/8d438dffe4834b66b7549ea3929d7623 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/8d438dffe4834b66b7549ea3929d7623 2024-11-11T23:53:52,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b9da267cf95b4ff6b8d33dd6fc427496 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b9da267cf95b4ff6b8d33dd6fc427496 2024-11-11T23:53:52,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0c355136003541709378c5a22b80ada8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/0c355136003541709378c5a22b80ada8 2024-11-11T23:53:52,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b8aa787b89a949f4a26c407765dd3529 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/b8aa787b89a949f4a26c407765dd3529 2024-11-11T23:53:52,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/455a74c1658f458c9e43cd0a2c2e01e2 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/455a74c1658f458c9e43cd0a2c2e01e2 2024-11-11T23:53:52,707 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/recovered.edits/576.seqid, newMaxSeqId=576, maxSeqId=1 2024-11-11T23:53:52,708 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. 2024-11-11T23:53:52,709 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 6b2061b8554b1e7422f6c037ae02cccd: Waiting for close lock at 1731369231271Running coprocessor pre-close hooks at 1731369231271Disabling compacts and flushes for region at 1731369231271Disabling writes for close at 1731369231272 (+1 ms)Obtaining lock to block concurrent updates at 1731369231272Preparing flush snapshotting stores in 6b2061b8554b1e7422f6c037ae02cccd at 1731369231272Finished memstore snapshotting TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731369231272Flushing stores of TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd. at 1731369231273 (+1 ms)Flushing 6b2061b8554b1e7422f6c037ae02cccd/A: creating writer at 1731369231273Flushing 6b2061b8554b1e7422f6c037ae02cccd/A: appending metadata at 1731369231278 (+5 ms)Flushing 6b2061b8554b1e7422f6c037ae02cccd/A: closing flushed file at 1731369231278Flushing 6b2061b8554b1e7422f6c037ae02cccd/B: creating writer at 1731369231692 (+414 ms)Flushing 6b2061b8554b1e7422f6c037ae02cccd/B: appending metadata at 1731369231694 (+2 ms)Flushing 6b2061b8554b1e7422f6c037ae02cccd/B: closing flushed file at 1731369231694Flushing 6b2061b8554b1e7422f6c037ae02cccd/C: creating writer at 1731369232113 (+419 ms)Flushing 6b2061b8554b1e7422f6c037ae02cccd/C: appending metadata at 1731369232116 (+3 ms)Flushing 6b2061b8554b1e7422f6c037ae02cccd/C: closing flushed file at 1731369232117 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7103d345: reopening flushed file at 1731369232533 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68d2994b: reopening flushed file at 1731369232539 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ac87d39: reopening flushed file at 1731369232544 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6b2061b8554b1e7422f6c037ae02cccd in 1278ms, sequenceid=573, compaction requested=false at 1731369232550 (+6 ms)Writing region close event to WAL at 1731369232704 (+154 ms)Running coprocessor post-close hooks at 1731369232708 (+4 ms)Closed at 1731369232708 2024-11-11T23:53:52,712 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:52,712 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6b2061b8554b1e7422f6c037ae02cccd, regionState=CLOSED 2024-11-11T23:53:52,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:53:52,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T23:53:52,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 6b2061b8554b1e7422f6c037ae02cccd, server=b70d29849faf,42393,1731369201430 in 1.6020 sec 2024-11-11T23:53:52,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-11T23:53:52,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b2061b8554b1e7422f6c037ae02cccd, UNASSIGN in 1.6120 sec 2024-11-11T23:53:52,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T23:53:52,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6200 sec 2024-11-11T23:53:52,723 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369232723"}]},"ts":"1731369232723"} 2024-11-11T23:53:52,725 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T23:53:52,725 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T23:53:52,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6400 sec 2024-11-11T23:53:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T23:53:53,245 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:53,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T23:53:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:53,255 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T23:53:53,258 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:53,265 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:53,270 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/recovered.edits] 2024-11-11T23:53:53,274 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/2a53d7ee735743f28728b5d6c6c937f3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/2a53d7ee735743f28728b5d6c6c937f3 2024-11-11T23:53:53,276 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a4288308c73b4260982c5d91135167a3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/A/a4288308c73b4260982c5d91135167a3 2024-11-11T23:53:53,280 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/28b45fced56545afb16814e91a1f83ee to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/28b45fced56545afb16814e91a1f83ee 2024-11-11T23:53:53,282 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/cc72c9f61f914e68a0062259bb0318b8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/B/cc72c9f61f914e68a0062259bb0318b8 2024-11-11T23:53:53,286 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/5982b2f081b54ce7b17e4ce9cccab757 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/5982b2f081b54ce7b17e4ce9cccab757 2024-11-11T23:53:53,288 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/e6ff7c2071f14f0488f0722a378d63e7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/C/e6ff7c2071f14f0488f0722a378d63e7 2024-11-11T23:53:53,292 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/recovered.edits/576.seqid to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd/recovered.edits/576.seqid 2024-11-11T23:53:53,293 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6b2061b8554b1e7422f6c037ae02cccd 2024-11-11T23:53:53,293 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T23:53:53,298 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:53,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T23:53:53,307 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T23:53:53,341 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T23:53:53,343 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:53,343 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T23:53:53,343 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731369233343"}]},"ts":"9223372036854775807"} 2024-11-11T23:53:53,348 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T23:53:53,348 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6b2061b8554b1e7422f6c037ae02cccd, NAME => 'TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T23:53:53,348 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T23:53:53,349 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731369233348"}]},"ts":"9223372036854775807"} 2024-11-11T23:53:53,357 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T23:53:53,358 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:53,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 109 msec 2024-11-11T23:53:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T23:53:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T23:53:53,366 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:53,379 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1248936657_22 at /127.0.0.1:39856 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b70d29849faf:42393-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_809876577_22 at /127.0.0.1:39872 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b70d29849faf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=547 (was 564), ProcessCount=11 (was 11), AvailableMemoryMB=4817 (was 5402) 2024-11-11T23:53:53,392 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=547, ProcessCount=11, AvailableMemoryMB=4816 2024-11-11T23:53:53,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:53:53,394 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:53:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:53,396 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:53:53,397 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:53,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-11T23:53:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T23:53:53,398 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:53:53,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741948_1124 (size=1061) 2024-11-11T23:53:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T23:53:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T23:53:53,813 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 2024-11-11T23:53:53,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741949_1125 (size=53) 2024-11-11T23:53:53,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:53,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d71f402ac6f3fd6fb0701033f3e67af7, disabling compactions & flushes 2024-11-11T23:53:53,832 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:53,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:53,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. after waiting 0 ms 2024-11-11T23:53:53,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:53,832 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:53,833 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d71f402ac6f3fd6fb0701033f3e67af7: Waiting for close lock at 1731369233832Disabling compacts and flushes for region at 1731369233832Disabling writes for close at 1731369233832Writing region close event to WAL at 1731369233832Closed at 1731369233832 2024-11-11T23:53:53,834 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:53:53,835 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731369233834"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369233834"}]},"ts":"1731369233834"} 2024-11-11T23:53:53,837 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:53:53,842 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:53:53,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369233842"}]},"ts":"1731369233842"} 2024-11-11T23:53:53,844 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T23:53:53,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, ASSIGN}] 2024-11-11T23:53:53,846 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, ASSIGN 2024-11-11T23:53:53,847 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, ASSIGN; state=OFFLINE, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=false 2024-11-11T23:53:53,998 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d71f402ac6f3fd6fb0701033f3e67af7, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:54,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, ASSIGN because future has completed 2024-11-11T23:53:54,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:53:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T23:53:54,160 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:54,160 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:53:54,161 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,161 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:54,161 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,161 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,163 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,165 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:54,166 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d71f402ac6f3fd6fb0701033f3e67af7 columnFamilyName A 2024-11-11T23:53:54,166 DEBUG [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:54,167 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(327): Store=d71f402ac6f3fd6fb0701033f3e67af7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:54,167 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,169 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:54,169 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d71f402ac6f3fd6fb0701033f3e67af7 columnFamilyName B 2024-11-11T23:53:54,169 DEBUG [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:54,170 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(327): Store=d71f402ac6f3fd6fb0701033f3e67af7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:54,170 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,171 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:54,171 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d71f402ac6f3fd6fb0701033f3e67af7 columnFamilyName C 2024-11-11T23:53:54,172 DEBUG [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:54,172 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(327): Store=d71f402ac6f3fd6fb0701033f3e67af7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:54,172 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:54,173 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,173 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,174 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,175 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,176 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,176 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:53:54,178 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,180 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:53:54,181 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened d71f402ac6f3fd6fb0701033f3e67af7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65584166, jitterRate=-0.022719770669937134}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:53:54,181 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:54,181 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for d71f402ac6f3fd6fb0701033f3e67af7: Running coprocessor pre-open hook at 1731369234161Writing region info on filesystem at 1731369234161Initializing all the Stores at 1731369234162 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369234163 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369234163Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369234163Cleaning up temporary data from old regions at 1731369234176 (+13 ms)Running coprocessor post-open hooks at 1731369234181 (+5 ms)Region opened successfully at 1731369234181 2024-11-11T23:53:54,182 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., pid=30, masterSystemTime=1731369234156 2024-11-11T23:53:54,184 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:54,184 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:54,184 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d71f402ac6f3fd6fb0701033f3e67af7, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:54,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:53:54,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T23:53:54,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 in 185 msec 2024-11-11T23:53:54,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-11T23:53:54,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, ASSIGN in 343 msec 2024-11-11T23:53:54,191 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:53:54,191 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369234191"}]},"ts":"1731369234191"} 2024-11-11T23:53:54,193 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T23:53:54,194 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:53:54,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 800 msec 2024-11-11T23:53:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T23:53:54,535 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:54,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:54,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:54,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:54,537 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681e6c75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:54,540 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93c960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:54,541 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:54,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:54,545 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T23:53:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b70d29849faf,37327,1731369200577 2024-11-11T23:53:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2022ca19 2024-11-11T23:53:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T23:53:54,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T23:53:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:53:54,554 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:53:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T23:53:54,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741950_1126 (size=1097) 2024-11-11T23:53:54,978 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T23:53:54,978 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T23:53:54,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:53:54,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, REOPEN/MOVE}] 2024-11-11T23:53:54,996 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, REOPEN/MOVE 2024-11-11T23:53:54,997 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d71f402ac6f3fd6fb0701033f3e67af7, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, REOPEN/MOVE because future has completed 2024-11-11T23:53:55,000 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:53:55,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:53:55,154 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,154 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:53:55,154 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing d71f402ac6f3fd6fb0701033f3e67af7, disabling compactions & flushes 2024-11-11T23:53:55,154 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,154 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,154 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. after waiting 0 ms 2024-11-11T23:53:55,154 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,158 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T23:53:55,159 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,159 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for d71f402ac6f3fd6fb0701033f3e67af7: Waiting for close lock at 1731369235154Running coprocessor pre-close hooks at 1731369235154Disabling compacts and flushes for region at 1731369235154Disabling writes for close at 1731369235154Writing region close event to WAL at 1731369235154Running coprocessor post-close hooks at 1731369235159 (+5 ms)Closed at 1731369235159 2024-11-11T23:53:55,159 WARN [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: d71f402ac6f3fd6fb0701033f3e67af7 to self. 2024-11-11T23:53:55,161 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,162 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d71f402ac6f3fd6fb0701033f3e67af7, regionState=CLOSED 2024-11-11T23:53:55,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:53:55,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T23:53:55,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 in 165 msec 2024-11-11T23:53:55,167 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, REOPEN/MOVE; state=CLOSED, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=true 2024-11-11T23:53:55,319 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d71f402ac6f3fd6fb0701033f3e67af7, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, REOPEN/MOVE because future has completed 2024-11-11T23:53:55,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:53:55,483 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,483 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:53:55,483 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,483 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:53:55,484 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,484 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,486 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,486 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:55,490 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d71f402ac6f3fd6fb0701033f3e67af7 columnFamilyName A 2024-11-11T23:53:55,492 DEBUG [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:55,492 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(327): Store=d71f402ac6f3fd6fb0701033f3e67af7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:55,493 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,493 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:55,494 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d71f402ac6f3fd6fb0701033f3e67af7 columnFamilyName B 2024-11-11T23:53:55,494 DEBUG [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:55,494 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(327): Store=d71f402ac6f3fd6fb0701033f3e67af7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:55,494 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,495 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:53:55,495 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d71f402ac6f3fd6fb0701033f3e67af7 columnFamilyName C 2024-11-11T23:53:55,495 DEBUG [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:55,496 INFO [StoreOpener-d71f402ac6f3fd6fb0701033f3e67af7-1 {}] regionserver.HStore(327): Store=d71f402ac6f3fd6fb0701033f3e67af7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:53:55,496 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,496 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,497 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,498 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,499 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,499 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,499 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:53:55,500 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,501 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened d71f402ac6f3fd6fb0701033f3e67af7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64993786, jitterRate=-0.03151711821556091}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:53:55,501 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,502 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for d71f402ac6f3fd6fb0701033f3e67af7: Running coprocessor pre-open hook at 1731369235484Writing region info on filesystem at 1731369235484Initializing all the Stores at 1731369235484Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369235484Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369235485 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369235485Cleaning up temporary data from old regions at 1731369235499 (+14 ms)Running coprocessor post-open hooks at 1731369235501 (+2 ms)Region opened successfully at 1731369235502 (+1 ms) 2024-11-11T23:53:55,503 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., pid=35, masterSystemTime=1731369235479 2024-11-11T23:53:55,505 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,505 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,505 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d71f402ac6f3fd6fb0701033f3e67af7, regionState=OPEN, openSeqNum=5, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:53:55,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-11T23:53:55,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 in 185 msec 2024-11-11T23:53:55,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-11T23:53:55,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, REOPEN/MOVE in 515 msec 2024-11-11T23:53:55,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T23:53:55,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-11T23:53:55,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 959 msec 2024-11-11T23:53:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T23:53:55,520 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:55,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,521 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c596ab2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,523 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1913bfac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,525 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb02826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,527 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22847ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,529 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b687ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@105a8cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,533 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c979fd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,535 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1625612, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,537 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7394917a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,538 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@762374ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,541 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e48489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,543 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4c3f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5655b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd05b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,552 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcc0eb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d31dbd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711c9f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:53:55,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:53:55,556 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:53:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:53:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:53:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2730ee6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:53:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:53:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:53:55,558 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:53:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594dbf65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:53:55,560 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,565 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:55,570 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,570 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,571 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,571 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,571 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,572 DEBUG [Thread-649 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,572 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,573 DEBUG [Thread-649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-11T23:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,574 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:53:55,575 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:55,576 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:55,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:53:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:53:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:53:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T23:53:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:53:55,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:53:55,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:53:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:53:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:53:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111115d3bf5b8d744859874b53eb231cd9ec_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369235594/Put/seqid=0 2024-11-11T23:53:55,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369295639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731369235619, max timestamp=1731369235652Num uniques 859; 2024-11-11T23:53:55,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731369235619, max timestamp=1731369235652Num uniques 859; 2024-11-11T23:53:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369295648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T23:53:55,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T23:53:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731369235619, max timestamp=1731369235652Num uniques 859; 2024-11-11T23:53:55,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T23:53:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741951_1127 (size=19474) 2024-11-11T23:53:55,677 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:55,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111115d3bf5b8d744859874b53eb231cd9ec_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111115d3bf5b8d744859874b53eb231cd9ec_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T23:53:55,693 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/3793848cf1da4a65992b64dcfe33d195, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:53:55,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369295692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/3793848cf1da4a65992b64dcfe33d195 is 175, key is test_row_0/A:col10/1731369235594/Put/seqid=0 2024-11-11T23:53:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369295693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369295702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741952_1128 (size=56733) 2024-11-11T23:53:55,722 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/3793848cf1da4a65992b64dcfe33d195 2024-11-11T23:53:55,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:55,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:55,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:55,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:55,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/6207fc4b3b0a4baaa597573de927436d is 50, key is test_row_0/B:col10/1731369235594/Put/seqid=0 2024-11-11T23:53:55,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369295765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369295765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741953_1129 (size=12001) 2024-11-11T23:53:55,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/6207fc4b3b0a4baaa597573de927436d 2024-11-11T23:53:55,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369295805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369295815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369295825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:55,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/fb1c9e735c7e4e9c8e38905ce2f3d16a is 50, key is test_row_0/C:col10/1731369235594/Put/seqid=0 2024-11-11T23:53:55,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741954_1130 (size=12001) 2024-11-11T23:53:55,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/fb1c9e735c7e4e9c8e38905ce2f3d16a 2024-11-11T23:53:55,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/3793848cf1da4a65992b64dcfe33d195 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3793848cf1da4a65992b64dcfe33d195 2024-11-11T23:53:55,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3793848cf1da4a65992b64dcfe33d195, entries=300, sequenceid=14, filesize=55.4 K 2024-11-11T23:53:55,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/6207fc4b3b0a4baaa597573de927436d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/6207fc4b3b0a4baaa597573de927436d 2024-11-11T23:53:55,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:55,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/6207fc4b3b0a4baaa597573de927436d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T23:53:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:55,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:55,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/fb1c9e735c7e4e9c8e38905ce2f3d16a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/fb1c9e735c7e4e9c8e38905ce2f3d16a 2024-11-11T23:53:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T23:53:55,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/fb1c9e735c7e4e9c8e38905ce2f3d16a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T23:53:55,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for d71f402ac6f3fd6fb0701033f3e67af7 in 305ms, sequenceid=14, compaction requested=false 2024-11-11T23:53:55,904 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T23:53:55,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:53:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:55,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=147.60 KB heapSize=396.70 KB 2024-11-11T23:53:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:53:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:53:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:53:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:56,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110fde8d2d3746406e8b8e6a192584fbfc_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369235672/Put/seqid=0 2024-11-11T23:53:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369296021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369296022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369296024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369296024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369296035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741955_1131 (size=14594) 2024-11-11T23:53:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:56,056 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110fde8d2d3746406e8b8e6a192584fbfc_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110fde8d2d3746406e8b8e6a192584fbfc_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:56,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/01563991f87c475f9f03ad274461770b, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:53:56,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/01563991f87c475f9f03ad274461770b is 175, key is test_row_0/A:col10/1731369235672/Put/seqid=0 2024-11-11T23:53:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741956_1132 (size=39549) 2024-11-11T23:53:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369296135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369296145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369296147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:56,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:56,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T23:53:56,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369296344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369296345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369296355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369296355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369296356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,494 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=47, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/01563991f87c475f9f03ad274461770b 2024-11-11T23:53:56,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/dc415fdc23cd4c8caf08d6a865e03a4c is 50, key is test_row_0/B:col10/1731369235672/Put/seqid=0 2024-11-11T23:53:56,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741957_1133 (size=12001) 2024-11-11T23:53:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369296654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369296675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369296675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T23:53:56,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:56,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:56,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369296864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369296875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:56,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/dc415fdc23cd4c8caf08d6a865e03a4c 2024-11-11T23:53:56,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/490c7d57f5464ab59938441f4387a08b is 50, key is test_row_0/C:col10/1731369235672/Put/seqid=0 2024-11-11T23:53:56,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:56,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:56,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:56,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:56,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741958_1134 (size=12001) 2024-11-11T23:53:56,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/490c7d57f5464ab59938441f4387a08b 2024-11-11T23:53:56,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/01563991f87c475f9f03ad274461770b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/01563991f87c475f9f03ad274461770b 2024-11-11T23:53:57,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/01563991f87c475f9f03ad274461770b, entries=200, sequenceid=47, filesize=38.6 K 2024-11-11T23:53:57,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/dc415fdc23cd4c8caf08d6a865e03a4c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/dc415fdc23cd4c8caf08d6a865e03a4c 2024-11-11T23:53:57,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/dc415fdc23cd4c8caf08d6a865e03a4c, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T23:53:57,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/490c7d57f5464ab59938441f4387a08b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/490c7d57f5464ab59938441f4387a08b 2024-11-11T23:53:57,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/490c7d57f5464ab59938441f4387a08b, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T23:53:57,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~396.66 KB/406176, currentSize=40.25 KB/41220 for d71f402ac6f3fd6fb0701033f3e67af7 in 1043ms, sequenceid=47, compaction requested=false 2024-11-11T23:53:57,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:53:57,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T23:53:57,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:57,129 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T23:53:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:53:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:53:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:53:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:53:57,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111127b6f72509ff46f2a14f1383d8eba6b1_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369236018/Put/seqid=0 2024-11-11T23:53:57,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741959_1135 (size=12154) 2024-11-11T23:53:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:57,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369237208, max timestamp=1731369237226Num uniques 859; 2024-11-11T23:53:57,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369237208, max timestamp=1731369237225Num uniques 859; 2024-11-11T23:53:57,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369237208, max timestamp=1731369237226Num uniques 859; 2024-11-11T23:53:57,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:57,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:57,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:53:57,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369237258, max timestamp=1731369237299Num uniques 1018; 2024-11-11T23:53:57,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369237258, max timestamp=1731369237299Num uniques 1018; 2024-11-11T23:53:57,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T23:53:57,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T23:53:57,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369237258, max timestamp=1731369237299Num uniques 1018; 2024-11-11T23:53:57,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T23:53:57,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369297384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369297388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369297391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369297495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369297497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369297508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:57,586 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111127b6f72509ff46f2a14f1383d8eba6b1_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111127b6f72509ff46f2a14f1383d8eba6b1_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:57,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/0b12338794f745a096350808222f76ed, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:53:57,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/0b12338794f745a096350808222f76ed is 175, key is test_row_0/A:col10/1731369236018/Put/seqid=0 2024-11-11T23:53:57,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741960_1136 (size=30955) 2024-11-11T23:53:57,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/0b12338794f745a096350808222f76ed 2024-11-11T23:53:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/90b920c6e64c469e920378db69a9721c is 50, key is test_row_0/B:col10/1731369236018/Put/seqid=0 2024-11-11T23:53:57,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741961_1137 (size=12001) 2024-11-11T23:53:57,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369297704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369297715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T23:53:57,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369297725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369297885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,886 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:57,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369297894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:57,896 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:58,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369298024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369298025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:58,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369298035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:58,080 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/90b920c6e64c469e920378db69a9721c 2024-11-11T23:53:58,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/f5ea4970c71a4365b5544b9abb07d80c is 50, key is test_row_0/C:col10/1731369236018/Put/seqid=0 2024-11-11T23:53:58,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741962_1138 (size=12001) 2024-11-11T23:53:58,242 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:53:58,530 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/f5ea4970c71a4365b5544b9abb07d80c 2024-11-11T23:53:58,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/0b12338794f745a096350808222f76ed as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/0b12338794f745a096350808222f76ed 2024-11-11T23:53:58,544 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/0b12338794f745a096350808222f76ed, entries=150, sequenceid=56, filesize=30.2 K 2024-11-11T23:53:58,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369298535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/90b920c6e64c469e920378db69a9721c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/90b920c6e64c469e920378db69a9721c 2024-11-11T23:53:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:58,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369298535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:58,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369298555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:58,564 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/90b920c6e64c469e920378db69a9721c, entries=150, sequenceid=56, filesize=11.7 K 2024-11-11T23:53:58,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/f5ea4970c71a4365b5544b9abb07d80c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/f5ea4970c71a4365b5544b9abb07d80c 2024-11-11T23:53:58,573 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/f5ea4970c71a4365b5544b9abb07d80c, entries=150, sequenceid=56, filesize=11.7 K 2024-11-11T23:53:58,575 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for d71f402ac6f3fd6fb0701033f3e67af7 in 1446ms, sequenceid=56, compaction requested=true 2024-11-11T23:53:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:53:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-11T23:53:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-11T23:53:58,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-11T23:53:58,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0010 sec 2024-11-11T23:53:58,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.0110 sec 2024-11-11T23:53:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:59,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=147.60 KB heapSize=414.61 KB 2024-11-11T23:53:59,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:53:59,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:59,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:53:59,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:59,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:53:59,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:53:59,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d1b4b1bc4dee4dd09e29f866843b31b3_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369237381/Put/seqid=0 2024-11-11T23:53:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369299578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369299579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369299580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741963_1139 (size=14594) 2024-11-11T23:53:59,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369299694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369299695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369299696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T23:53:59,744 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:53:59,745 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:53:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-11T23:53:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T23:53:59,748 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:53:59,749 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:53:59,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:53:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T23:53:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369299894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,896 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:53:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:53:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:53:59,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:53:59,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369299904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369299905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369299905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:53:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369299914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:53:59,916 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:53:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:53:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:53:59,991 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:53:59,997 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d1b4b1bc4dee4dd09e29f866843b31b3_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d1b4b1bc4dee4dd09e29f866843b31b3_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:53:59,998 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/e2370ebb7dbf4ffba18e3ade07e77a8d, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:53:59,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/e2370ebb7dbf4ffba18e3ade07e77a8d is 175, key is test_row_0/A:col10/1731369237381/Put/seqid=0 2024-11-11T23:54:00,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741964_1140 (size=39549) 2024-11-11T23:54:00,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:00,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:00,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T23:54:00,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369300215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369300215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:00,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369300215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:00,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T23:54:00,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/e2370ebb7dbf4ffba18e3ade07e77a8d 2024-11-11T23:54:00,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/4ab6229a75114ef695be4a8adc0fc9be is 50, key is test_row_0/B:col10/1731369237381/Put/seqid=0 2024-11-11T23:54:00,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741965_1141 (size=12001) 2024-11-11T23:54:00,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:00,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:00,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:00,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:00,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369300724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369300724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:00,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369300724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:00,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/4ab6229a75114ef695be4a8adc0fc9be 2024-11-11T23:54:00,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/0c0fb5b971ed49ada5aefb6872cd3446 is 50, key is test_row_0/C:col10/1731369237381/Put/seqid=0 2024-11-11T23:54:00,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741966_1142 (size=12001) 2024-11-11T23:54:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T23:54:00,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:00,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:01,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:01,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:01,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/0c0fb5b971ed49ada5aefb6872cd3446 2024-11-11T23:54:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/e2370ebb7dbf4ffba18e3ade07e77a8d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e2370ebb7dbf4ffba18e3ade07e77a8d 2024-11-11T23:54:01,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e2370ebb7dbf4ffba18e3ade07e77a8d, entries=200, sequenceid=110, filesize=38.6 K 2024-11-11T23:54:01,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/4ab6229a75114ef695be4a8adc0fc9be as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4ab6229a75114ef695be4a8adc0fc9be 2024-11-11T23:54:01,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4ab6229a75114ef695be4a8adc0fc9be, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T23:54:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/0c0fb5b971ed49ada5aefb6872cd3446 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/0c0fb5b971ed49ada5aefb6872cd3446 2024-11-11T23:54:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/0c0fb5b971ed49ada5aefb6872cd3446, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T23:54:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.56 KB/424512, currentSize=33.54 KB/34350 for d71f402ac6f3fd6fb0701033f3e67af7 in 1710ms, sequenceid=110, compaction requested=true 2024-11-11T23:54:01,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:01,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:01,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:01,270 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:01,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:01,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:01,270 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:01,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:01,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,272 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:01,272 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/B is initiating minor compaction (all files) 2024-11-11T23:54:01,272 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/B in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,272 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/6207fc4b3b0a4baaa597573de927436d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/dc415fdc23cd4c8caf08d6a865e03a4c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/90b920c6e64c469e920378db69a9721c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4ab6229a75114ef695be4a8adc0fc9be] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=46.9 K 2024-11-11T23:54:01,272 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166786 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,272 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/A is initiating minor compaction (all files) 2024-11-11T23:54:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,272 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/A in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:01,273 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3793848cf1da4a65992b64dcfe33d195, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/01563991f87c475f9f03ad274461770b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/0b12338794f745a096350808222f76ed, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e2370ebb7dbf4ffba18e3ade07e77a8d] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=162.9 K 2024-11-11T23:54:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,273 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:01,273 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3793848cf1da4a65992b64dcfe33d195, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/01563991f87c475f9f03ad274461770b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/0b12338794f745a096350808222f76ed, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e2370ebb7dbf4ffba18e3ade07e77a8d] 2024-11-11T23:54:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,274 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6207fc4b3b0a4baaa597573de927436d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731369235580 2024-11-11T23:54:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,274 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3793848cf1da4a65992b64dcfe33d195, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731369235570 2024-11-11T23:54:01,274 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting dc415fdc23cd4c8caf08d6a865e03a4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731369235672 2024-11-11T23:54:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,275 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01563991f87c475f9f03ad274461770b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731369235672 2024-11-11T23:54:01,275 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b920c6e64c469e920378db69a9721c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731369236008 2024-11-11T23:54:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,275 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ab6229a75114ef695be4a8adc0fc9be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731369237381 2024-11-11T23:54:01,275 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b12338794f745a096350808222f76ed, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731369236008 2024-11-11T23:54:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,277 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2370ebb7dbf4ffba18e3ade07e77a8d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731369237369 2024-11-11T23:54:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T23:54:01,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:01,286 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T23:54:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:01,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:01,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:01,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:01,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:01,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,291 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,294 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#B#compaction#123 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,295 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/43fb5f379335446e8132ebbbb75c4ddd is 50, key is test_row_0/B:col10/1731369237381/Put/seqid=0 2024-11-11T23:54:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,302 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c12d81b83af84194b7b05a0b7ce21c0a_d71f402ac6f3fd6fb0701033f3e67af7 store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,306 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c12d81b83af84194b7b05a0b7ce21c0a_d71f402ac6f3fd6fb0701033f3e67af7, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:01,307 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c12d81b83af84194b7b05a0b7ce21c0a_d71f402ac6f3fd6fb0701033f3e67af7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116ae8037726084380a396ecb247a3d696_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369239572/Put/seqid=0 2024-11-11T23:54:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741967_1143 (size=12139) 2024-11-11T23:54:01,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741968_1144 (size=4469) 2024-11-11T23:54:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741969_1145 (size=9714) 2024-11-11T23:54:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,359 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116ae8037726084380a396ecb247a3d696_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ae8037726084380a396ecb247a3d696_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/d5a71d6c40be4437bd6a6497c61febe7, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/d5a71d6c40be4437bd6a6497c61febe7 is 175, key is test_row_0/A:col10/1731369239572/Put/seqid=0 2024-11-11T23:54:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741970_1146 (size=22361) 2024-11-11T23:54:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,382 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/d5a71d6c40be4437bd6a6497c61febe7 2024-11-11T23:54:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/70bb24c8c53f41f383a258561c164476 is 50, key is test_row_0/B:col10/1731369239572/Put/seqid=0 2024-11-11T23:54:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741971_1147 (size=9657) 2024-11-11T23:54:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,735 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#A#compaction#124 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,736 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/610d3387f1b847929084b5adc835ff19 is 175, key is test_row_0/A:col10/1731369237381/Put/seqid=0 2024-11-11T23:54:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,741 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/43fb5f379335446e8132ebbbb75c4ddd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/43fb5f379335446e8132ebbbb75c4ddd 2024-11-11T23:54:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,751 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/B of d71f402ac6f3fd6fb0701033f3e67af7 into 43fb5f379335446e8132ebbbb75c4ddd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:01,751 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:01,751 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/B, priority=12, startTime=1731369241270; duration=0sec 2024-11-11T23:54:01,751 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:01,751 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:B 2024-11-11T23:54:01,751 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,753 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:01,753 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/C is initiating minor compaction (all files) 2024-11-11T23:54:01,753 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/C in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:01,753 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/fb1c9e735c7e4e9c8e38905ce2f3d16a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/490c7d57f5464ab59938441f4387a08b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/f5ea4970c71a4365b5544b9abb07d80c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/0c0fb5b971ed49ada5aefb6872cd3446] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=46.9 K 2024-11-11T23:54:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,754 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting fb1c9e735c7e4e9c8e38905ce2f3d16a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731369235580 2024-11-11T23:54:01,754 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 490c7d57f5464ab59938441f4387a08b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731369235672 2024-11-11T23:54:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,755 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f5ea4970c71a4365b5544b9abb07d80c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731369236008 2024-11-11T23:54:01,756 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c0fb5b971ed49ada5aefb6872cd3446, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731369237381 2024-11-11T23:54:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741972_1148 (size=31093) 2024-11-11T23:54:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,775 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/610d3387f1b847929084b5adc835ff19 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/610d3387f1b847929084b5adc835ff19 2024-11-11T23:54:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,782 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/A of d71f402ac6f3fd6fb0701033f3e67af7 into 610d3387f1b847929084b5adc835ff19(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:01,782 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:01,783 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/A, priority=12, startTime=1731369241270; duration=0sec 2024-11-11T23:54:01,783 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:01,783 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:A 2024-11-11T23:54:01,784 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#C#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:01,784 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/8142cdfd5b004ae6b6f4928529bd4bf3 is 50, key is test_row_0/C:col10/1731369237381/Put/seqid=0 2024-11-11T23:54:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741973_1149 (size=12139) 2024-11-11T23:54:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:01,799 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/8142cdfd5b004ae6b6f4928529bd4bf3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8142cdfd5b004ae6b6f4928529bd4bf3 2024-11-11T23:54:01,808 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/C of d71f402ac6f3fd6fb0701033f3e67af7 into 8142cdfd5b004ae6b6f4928529bd4bf3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:01,808 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:01,809 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/C, priority=12, startTime=1731369241270; duration=0sec 2024-11-11T23:54:01,809 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:01,809 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:C 2024-11-11T23:54:01,829 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/70bb24c8c53f41f383a258561c164476 2024-11-11T23:54:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/366d96c3068a478888424024341c27ac is 50, key is test_row_0/C:col10/1731369239572/Put/seqid=0 2024-11-11T23:54:01,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741974_1150 (size=9657) 2024-11-11T23:54:01,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369241818, max timestamp=1731369241822Num uniques 859; 2024-11-11T23:54:01,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369241818, max timestamp=1731369241822Num uniques 859; 2024-11-11T23:54:01,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:01,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:01,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369241818, max timestamp=1731369241822Num uniques 859; 2024-11-11T23:54:01,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T23:54:01,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369241885, max timestamp=1731369241916Num uniques 1018; 2024-11-11T23:54:01,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T23:54:01,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731369241905, max timestamp=1731369241916Num uniques 1018; 2024-11-11T23:54:01,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-11T23:54:01,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369241885, max timestamp=1731369241916Num uniques 1018; 2024-11-11T23:54:01,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T23:54:01,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731369241966, max timestamp=1731369241980Num uniques 1127; 2024-11-11T23:54:01,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-11T23:54:01,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731369241966, max timestamp=1731369241980Num uniques 1009; 2024-11-11T23:54:01,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731369241966, max timestamp=1731369241980Num uniques 1127; 2024-11-11T23:54:01,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-11T23:54:01,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-11-11T23:54:02,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369302022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369302024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369302028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369302135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369302135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369302144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,251 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/366d96c3068a478888424024341c27ac 2024-11-11T23:54:02,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/d5a71d6c40be4437bd6a6497c61febe7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d5a71d6c40be4437bd6a6497c61febe7 2024-11-11T23:54:02,271 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d5a71d6c40be4437bd6a6497c61febe7, entries=100, sequenceid=118, filesize=21.8 K 2024-11-11T23:54:02,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/70bb24c8c53f41f383a258561c164476 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/70bb24c8c53f41f383a258561c164476 2024-11-11T23:54:02,283 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/70bb24c8c53f41f383a258561c164476, entries=100, sequenceid=118, filesize=9.4 K 2024-11-11T23:54:02,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/366d96c3068a478888424024341c27ac as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/366d96c3068a478888424024341c27ac 2024-11-11T23:54:02,289 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/366d96c3068a478888424024341c27ac, entries=100, sequenceid=118, filesize=9.4 K 2024-11-11T23:54:02,291 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for d71f402ac6f3fd6fb0701033f3e67af7 in 1005ms, sequenceid=118, compaction requested=false 2024-11-11T23:54:02,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:02,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:02,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-11T23:54:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-11T23:54:02,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T23:54:02,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5430 sec 2024-11-11T23:54:02,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.5510 sec 2024-11-11T23:54:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:02,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731369242019, max timestamp=1731369242348Num uniques 1136; 2024-11-11T23:54:02,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731369242019, max timestamp=1731369242348Num uniques 1136; 2024-11-11T23:54:02,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-11T23:54:02,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-11T23:54:02,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=69.33 KB heapSize=187.27 KB 2024-11-11T23:54:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:54:02,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111155b347e989834cefb13ccc51721014fc_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369242352/Put/seqid=0 2024-11-11T23:54:02,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369302387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369302388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369302390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741975_1151 (size=17284) 2024-11-11T23:54:02,405 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:02,410 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111155b347e989834cefb13ccc51721014fc_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155b347e989834cefb13ccc51721014fc_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:02,411 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/2e32a05fe1bd41b6bc554ff08008bd87, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:02,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/2e32a05fe1bd41b6bc554ff08008bd87 is 175, key is test_row_0/A:col10/1731369242352/Put/seqid=0 2024-11-11T23:54:02,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741976_1152 (size=48389) 2024-11-11T23:54:02,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/2e32a05fe1bd41b6bc554ff08008bd87 2024-11-11T23:54:02,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/572497a1c8bd48249e80cd8fec1bdcb4 is 50, key is test_row_0/B:col10/1731369242352/Put/seqid=0 2024-11-11T23:54:02,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741977_1153 (size=12151) 2024-11-11T23:54:02,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369302505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369302505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369302505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369302715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369302715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369302715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:02,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/572497a1c8bd48249e80cd8fec1bdcb4 2024-11-11T23:54:02,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/344bdf59367542b7b9167f63aabdf2f8 is 50, key is test_row_0/C:col10/1731369242352/Put/seqid=0 2024-11-11T23:54:02,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741978_1154 (size=12151) 2024-11-11T23:54:02,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/344bdf59367542b7b9167f63aabdf2f8 2024-11-11T23:54:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/2e32a05fe1bd41b6bc554ff08008bd87 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/2e32a05fe1bd41b6bc554ff08008bd87 2024-11-11T23:54:02,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/2e32a05fe1bd41b6bc554ff08008bd87, entries=250, sequenceid=194, filesize=47.3 K 2024-11-11T23:54:02,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/572497a1c8bd48249e80cd8fec1bdcb4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/572497a1c8bd48249e80cd8fec1bdcb4 2024-11-11T23:54:02,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/572497a1c8bd48249e80cd8fec1bdcb4, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T23:54:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/344bdf59367542b7b9167f63aabdf2f8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/344bdf59367542b7b9167f63aabdf2f8 2024-11-11T23:54:02,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/344bdf59367542b7b9167f63aabdf2f8, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T23:54:02,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.04 KB/77860, heapSize ~207.14 KB/212112, currentSize=107.34 KB/109920 for d71f402ac6f3fd6fb0701033f3e67af7 in 539ms, sequenceid=194, compaction requested=true 2024-11-11T23:54:02,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:02,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:02,892 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:02,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:02,893 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:02,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:02,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:02,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:02,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:02,894 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:02,894 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:02,894 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/A is initiating minor compaction (all files) 2024-11-11T23:54:02,894 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/B is initiating minor compaction (all files) 2024-11-11T23:54:02,894 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/A in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:02,894 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/B in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:02,895 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/610d3387f1b847929084b5adc835ff19, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d5a71d6c40be4437bd6a6497c61febe7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/2e32a05fe1bd41b6bc554ff08008bd87] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=99.5 K 2024-11-11T23:54:02,895 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/43fb5f379335446e8132ebbbb75c4ddd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/70bb24c8c53f41f383a258561c164476, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/572497a1c8bd48249e80cd8fec1bdcb4] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=33.2 K 2024-11-11T23:54:02,895 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:02,895 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/610d3387f1b847929084b5adc835ff19, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d5a71d6c40be4437bd6a6497c61febe7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/2e32a05fe1bd41b6bc554ff08008bd87] 2024-11-11T23:54:02,895 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 43fb5f379335446e8132ebbbb75c4ddd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731369237381 2024-11-11T23:54:02,895 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 70bb24c8c53f41f383a258561c164476, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731369239572 2024-11-11T23:54:02,895 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 610d3387f1b847929084b5adc835ff19, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731369237381 2024-11-11T23:54:02,896 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a71d6c40be4437bd6a6497c61febe7, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731369239572 2024-11-11T23:54:02,896 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 572497a1c8bd48249e80cd8fec1bdcb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731369242024 2024-11-11T23:54:02,896 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e32a05fe1bd41b6bc554ff08008bd87, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731369242019 2024-11-11T23:54:02,915 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:02,916 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/ca54f03aeaf141fab0892a1dbf33ef0f is 50, key is test_row_0/B:col10/1731369242352/Put/seqid=0 2024-11-11T23:54:02,916 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:02,919 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111199bf15d2c71f4d2cbd70072c846f63b9_d71f402ac6f3fd6fb0701033f3e67af7 store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:02,924 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111199bf15d2c71f4d2cbd70072c846f63b9_d71f402ac6f3fd6fb0701033f3e67af7, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:02,924 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111199bf15d2c71f4d2cbd70072c846f63b9_d71f402ac6f3fd6fb0701033f3e67af7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:02,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741979_1155 (size=12391) 2024-11-11T23:54:02,941 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/ca54f03aeaf141fab0892a1dbf33ef0f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/ca54f03aeaf141fab0892a1dbf33ef0f 2024-11-11T23:54:02,947 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/B of d71f402ac6f3fd6fb0701033f3e67af7 into ca54f03aeaf141fab0892a1dbf33ef0f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:02,947 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:02,947 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/B, priority=13, startTime=1731369242892; duration=0sec 2024-11-11T23:54:02,947 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:02,947 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:B 2024-11-11T23:54:02,948 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:02,949 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:02,950 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/C is initiating minor compaction (all files) 2024-11-11T23:54:02,950 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/C in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:02,950 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8142cdfd5b004ae6b6f4928529bd4bf3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/366d96c3068a478888424024341c27ac, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/344bdf59367542b7b9167f63aabdf2f8] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=33.2 K 2024-11-11T23:54:02,952 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8142cdfd5b004ae6b6f4928529bd4bf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731369237381 2024-11-11T23:54:02,953 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 366d96c3068a478888424024341c27ac, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731369239572 2024-11-11T23:54:02,953 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 344bdf59367542b7b9167f63aabdf2f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731369242024 2024-11-11T23:54:02,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741980_1156 (size=4469) 2024-11-11T23:54:02,975 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#C#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:02,976 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/dcc117440f8d4ceca53189e8ab39f327 is 50, key is test_row_0/C:col10/1731369242352/Put/seqid=0 2024-11-11T23:54:02,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741981_1157 (size=12391) 2024-11-11T23:54:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:03,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T23:54:03,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:03,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:03,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:03,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:03,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:03,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:03,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369303038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f89c52d5fb794d51ae7a48bb61dde00c_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369242386/Put/seqid=0 2024-11-11T23:54:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369303039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369303040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741982_1158 (size=17284) 2024-11-11T23:54:03,055 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:03,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f89c52d5fb794d51ae7a48bb61dde00c_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f89c52d5fb794d51ae7a48bb61dde00c_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:03,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/c997f60c0c1f41979bec817b80dac9c7, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:03,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/c997f60c0c1f41979bec817b80dac9c7 is 175, key is test_row_0/A:col10/1731369242386/Put/seqid=0 2024-11-11T23:54:03,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741983_1159 (size=48389) 2024-11-11T23:54:03,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369303145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369303154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369303154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369303355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,364 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#A#compaction#133 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:03,364 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/3d947a462580496eb17a583aa3cd85b5 is 175, key is test_row_0/A:col10/1731369242352/Put/seqid=0 2024-11-11T23:54:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369303364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369303365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741984_1160 (size=31345) 2024-11-11T23:54:03,387 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/3d947a462580496eb17a583aa3cd85b5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3d947a462580496eb17a583aa3cd85b5 2024-11-11T23:54:03,393 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/dcc117440f8d4ceca53189e8ab39f327 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/dcc117440f8d4ceca53189e8ab39f327 2024-11-11T23:54:03,395 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/A of d71f402ac6f3fd6fb0701033f3e67af7 into 3d947a462580496eb17a583aa3cd85b5(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:03,395 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:03,395 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/A, priority=13, startTime=1731369242892; duration=0sec 2024-11-11T23:54:03,395 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:03,395 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:A 2024-11-11T23:54:03,401 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/C of d71f402ac6f3fd6fb0701033f3e67af7 into dcc117440f8d4ceca53189e8ab39f327(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:03,401 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:03,401 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/C, priority=13, startTime=1731369242894; duration=0sec 2024-11-11T23:54:03,401 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:03,401 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:C 2024-11-11T23:54:03,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/c997f60c0c1f41979bec817b80dac9c7 2024-11-11T23:54:03,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/a145dd2d7dfe4872b598c22da1816723 is 50, key is test_row_0/B:col10/1731369242386/Put/seqid=0 2024-11-11T23:54:03,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741985_1161 (size=12151) 2024-11-11T23:54:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369303665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369303674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369303674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/a145dd2d7dfe4872b598c22da1816723 2024-11-11T23:54:03,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/aab84cd12759469086bec5091a37748f is 50, key is test_row_0/C:col10/1731369242386/Put/seqid=0 2024-11-11T23:54:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T23:54:03,905 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:03,906 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T23:54:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T23:54:03,908 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:03,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:03,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:03,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741986_1162 (size=12151) 2024-11-11T23:54:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369303924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,927 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:03,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369303934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:03,939 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T23:54:04,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T23:54:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:04,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369304184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369304184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369304185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:04,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T23:54:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:04,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T23:54:04,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/aab84cd12759469086bec5091a37748f 2024-11-11T23:54:04,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/c997f60c0c1f41979bec817b80dac9c7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/c997f60c0c1f41979bec817b80dac9c7 2024-11-11T23:54:04,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/c997f60c0c1f41979bec817b80dac9c7, entries=250, sequenceid=216, filesize=47.3 K 2024-11-11T23:54:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/a145dd2d7dfe4872b598c22da1816723 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a145dd2d7dfe4872b598c22da1816723 2024-11-11T23:54:04,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a145dd2d7dfe4872b598c22da1816723, entries=150, sequenceid=216, filesize=11.9 K 2024-11-11T23:54:04,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/aab84cd12759469086bec5091a37748f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/aab84cd12759469086bec5091a37748f 2024-11-11T23:54:04,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/aab84cd12759469086bec5091a37748f, entries=150, sequenceid=216, filesize=11.9 K 2024-11-11T23:54:04,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d71f402ac6f3fd6fb0701033f3e67af7 in 1312ms, sequenceid=216, compaction requested=false 2024-11-11T23:54:04,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:04,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T23:54:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:04,369 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T23:54:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:04,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113d74e4d1b8f34b1c8f6c498c282e5e18_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369243039/Put/seqid=0 2024-11-11T23:54:04,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741987_1163 (size=12304) 2024-11-11T23:54:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T23:54:04,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:04,800 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113d74e4d1b8f34b1c8f6c498c282e5e18_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113d74e4d1b8f34b1c8f6c498c282e5e18_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:04,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/8e42eedba3dc45bf89e6587b13a15af2, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:04,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/8e42eedba3dc45bf89e6587b13a15af2 is 175, key is test_row_0/A:col10/1731369243039/Put/seqid=0 2024-11-11T23:54:04,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741988_1164 (size=31105) 2024-11-11T23:54:04,810 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/8e42eedba3dc45bf89e6587b13a15af2 2024-11-11T23:54:04,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/0fb97953b7f14c488e908dd92be95da1 is 50, key is test_row_0/B:col10/1731369243039/Put/seqid=0 2024-11-11T23:54:04,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741989_1165 (size=12151) 2024-11-11T23:54:04,984 INFO [master/b70d29849faf:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T23:54:04,984 INFO [master/b70d29849faf:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T23:54:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T23:54:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:05,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369305225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369305229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369245197, max timestamp=1731369245228Num uniques 859; 2024-11-11T23:54:05,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:54:05,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369245197, max timestamp=1731369245228Num uniques 859; 2024-11-11T23:54:05,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369245197, max timestamp=1731369245228Num uniques 859; 2024-11-11T23:54:05,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:54:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/0fb97953b7f14c488e908dd92be95da1 2024-11-11T23:54:05,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c02a47c6d4e84d588bdbd8e3b9269b22 is 50, key is test_row_0/C:col10/1731369243039/Put/seqid=0 2024-11-11T23:54:05,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741990_1166 (size=12151) 2024-11-11T23:54:05,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369305310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369305335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369305335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369305425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369305545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369305545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369305635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,674 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c02a47c6d4e84d588bdbd8e3b9269b22 2024-11-11T23:54:05,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/8e42eedba3dc45bf89e6587b13a15af2 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/8e42eedba3dc45bf89e6587b13a15af2 2024-11-11T23:54:05,683 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/8e42eedba3dc45bf89e6587b13a15af2, entries=150, sequenceid=229, filesize=30.4 K 2024-11-11T23:54:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/0fb97953b7f14c488e908dd92be95da1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/0fb97953b7f14c488e908dd92be95da1 2024-11-11T23:54:05,689 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/0fb97953b7f14c488e908dd92be95da1, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T23:54:05,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c02a47c6d4e84d588bdbd8e3b9269b22 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c02a47c6d4e84d588bdbd8e3b9269b22 2024-11-11T23:54:05,694 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c02a47c6d4e84d588bdbd8e3b9269b22, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T23:54:05,695 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d71f402ac6f3fd6fb0701033f3e67af7 in 1326ms, sequenceid=229, compaction requested=true 2024-11-11T23:54:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T23:54:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T23:54:05,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T23:54:05,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7880 sec 2024-11-11T23:54:05,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7950 sec 2024-11-11T23:54:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:05,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T23:54:05,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:05,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:05,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:05,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:05,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:05,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:05,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d1db13789c6d4129b80c3d80d6f28599_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369245273/Put/seqid=0 2024-11-11T23:54:05,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741991_1167 (size=14994) 2024-11-11T23:54:05,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369305884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369305885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369305945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369305995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369305995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T23:54:06,054 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:06,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T23:54:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T23:54:06,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:06,058 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:06,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T23:54:06,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369306204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:06,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369306205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:06,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:06,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:06,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:06,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d1db13789c6d4129b80c3d80d6f28599_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d1db13789c6d4129b80c3d80d6f28599_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:06,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/e280818aa2bf4a06a04af320fd5dcd1f, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:06,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/e280818aa2bf4a06a04af320fd5dcd1f is 175, key is test_row_0/A:col10/1731369245273/Put/seqid=0 2024-11-11T23:54:06,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741992_1168 (size=39949) 2024-11-11T23:54:06,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:06,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:06,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T23:54:06,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369306465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:06,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369306514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:06,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369306514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:06,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:06,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T23:54:06,691 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/e280818aa2bf4a06a04af320fd5dcd1f 2024-11-11T23:54:06,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/96fbeab0f9f24c428fcca75902567880 is 50, key is test_row_0/B:col10/1731369245273/Put/seqid=0 2024-11-11T23:54:06,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741993_1169 (size=12301) 2024-11-11T23:54:06,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:06,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:06,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:06,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:06,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:06,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369307024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369307034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:07,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/96fbeab0f9f24c428fcca75902567880 2024-11-11T23:54:07,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/8f30ef0be25d4c1f96c5b3fb090e1c72 is 50, key is test_row_0/C:col10/1731369245273/Put/seqid=0 2024-11-11T23:54:07,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:07,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:07,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741994_1170 (size=12301) 2024-11-11T23:54:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T23:54:07,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:07,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:07,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:07,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369307484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:07,491 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:07,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/8f30ef0be25d4c1f96c5b3fb090e1c72 2024-11-11T23:54:07,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/e280818aa2bf4a06a04af320fd5dcd1f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e280818aa2bf4a06a04af320fd5dcd1f 2024-11-11T23:54:07,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e280818aa2bf4a06a04af320fd5dcd1f, entries=200, sequenceid=265, filesize=39.0 K 2024-11-11T23:54:07,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/96fbeab0f9f24c428fcca75902567880 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/96fbeab0f9f24c428fcca75902567880 2024-11-11T23:54:07,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/96fbeab0f9f24c428fcca75902567880, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T23:54:07,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/8f30ef0be25d4c1f96c5b3fb090e1c72 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8f30ef0be25d4c1f96c5b3fb090e1c72 2024-11-11T23:54:07,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8f30ef0be25d4c1f96c5b3fb090e1c72, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T23:54:07,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for d71f402ac6f3fd6fb0701033f3e67af7 in 1701ms, sequenceid=265, compaction requested=true 2024-11-11T23:54:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:07,560 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:07,560 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:07,562 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150788 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:07,562 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48994 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:07,562 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/B is initiating minor compaction (all files) 2024-11-11T23:54:07,562 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/A is initiating minor compaction (all files) 2024-11-11T23:54:07,562 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/B in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,562 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/ca54f03aeaf141fab0892a1dbf33ef0f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a145dd2d7dfe4872b598c22da1816723, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/0fb97953b7f14c488e908dd92be95da1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/96fbeab0f9f24c428fcca75902567880] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=47.8 K 2024-11-11T23:54:07,562 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/A in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,562 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3d947a462580496eb17a583aa3cd85b5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/c997f60c0c1f41979bec817b80dac9c7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/8e42eedba3dc45bf89e6587b13a15af2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e280818aa2bf4a06a04af320fd5dcd1f] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=147.3 K 2024-11-11T23:54:07,562 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,562 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3d947a462580496eb17a583aa3cd85b5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/c997f60c0c1f41979bec817b80dac9c7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/8e42eedba3dc45bf89e6587b13a15af2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e280818aa2bf4a06a04af320fd5dcd1f] 2024-11-11T23:54:07,563 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ca54f03aeaf141fab0892a1dbf33ef0f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731369242024 2024-11-11T23:54:07,563 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d947a462580496eb17a583aa3cd85b5, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731369242024 2024-11-11T23:54:07,564 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c997f60c0c1f41979bec817b80dac9c7, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731369242384 2024-11-11T23:54:07,564 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a145dd2d7dfe4872b598c22da1816723, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731369242386 2024-11-11T23:54:07,564 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fb97953b7f14c488e908dd92be95da1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731369243034 2024-11-11T23:54:07,564 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e42eedba3dc45bf89e6587b13a15af2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731369243034 2024-11-11T23:54:07,564 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e280818aa2bf4a06a04af320fd5dcd1f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369245273 2024-11-11T23:54:07,564 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 96fbeab0f9f24c428fcca75902567880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369245273 2024-11-11T23:54:07,577 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#B#compaction#144 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:07,577 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:07,577 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/a31bfa55ca6a4feba9bd051f74d3dc88 is 50, key is test_row_0/B:col10/1731369245273/Put/seqid=0 2024-11-11T23:54:07,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T23:54:07,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:07,590 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T23:54:07,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:07,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:07,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:07,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:07,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:07,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:07,606 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411117f35d70704844af3b9d0164e903a55b8_d71f402ac6f3fd6fb0701033f3e67af7 store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:07,608 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411117f35d70704844af3b9d0164e903a55b8_d71f402ac6f3fd6fb0701033f3e67af7, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:07,609 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117f35d70704844af3b9d0164e903a55b8_d71f402ac6f3fd6fb0701033f3e67af7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:07,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741995_1171 (size=12677) 2024-11-11T23:54:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111b7dbd3646354a1fa202bbea3aa938f3_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369245861/Put/seqid=0 2024-11-11T23:54:07,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741997_1173 (size=12454) 2024-11-11T23:54:07,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741996_1172 (size=4469) 2024-11-11T23:54:07,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:07,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111b7dbd3646354a1fa202bbea3aa938f3_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b7dbd3646354a1fa202bbea3aa938f3_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:07,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/be963be5e469415183f7250edc0f82b9, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:07,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/be963be5e469415183f7250edc0f82b9 is 175, key is test_row_0/A:col10/1731369245861/Put/seqid=0 2024-11-11T23:54:07,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741998_1174 (size=31255) 2024-11-11T23:54:08,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:08,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:08,043 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/a31bfa55ca6a4feba9bd051f74d3dc88 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a31bfa55ca6a4feba9bd051f74d3dc88 2024-11-11T23:54:08,044 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#A#compaction#145 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:08,045 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/d868a62c9e9649409a5d7afae0f75491 is 175, key is test_row_0/A:col10/1731369245273/Put/seqid=0 2024-11-11T23:54:08,060 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/be963be5e469415183f7250edc0f82b9 2024-11-11T23:54:08,062 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/B of d71f402ac6f3fd6fb0701033f3e67af7 into a31bfa55ca6a4feba9bd051f74d3dc88(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:08,062 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:08,062 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/B, priority=12, startTime=1731369247560; duration=0sec 2024-11-11T23:54:08,062 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:08,063 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:B 2024-11-11T23:54:08,063 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:08,066 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48994 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:08,067 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/C is initiating minor compaction (all files) 2024-11-11T23:54:08,068 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/C in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:08,068 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/dcc117440f8d4ceca53189e8ab39f327, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/aab84cd12759469086bec5091a37748f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c02a47c6d4e84d588bdbd8e3b9269b22, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8f30ef0be25d4c1f96c5b3fb090e1c72] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=47.8 K 2024-11-11T23:54:08,068 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc117440f8d4ceca53189e8ab39f327, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731369242024 2024-11-11T23:54:08,069 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting aab84cd12759469086bec5091a37748f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731369242386 2024-11-11T23:54:08,073 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c02a47c6d4e84d588bdbd8e3b9269b22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731369243034 2024-11-11T23:54:08,074 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f30ef0be25d4c1f96c5b3fb090e1c72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369245273 2024-11-11T23:54:08,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/1ed615d58c2741ac9c9f8861d704cd6b is 50, key is test_row_0/B:col10/1731369245861/Put/seqid=0 2024-11-11T23:54:08,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741999_1175 (size=31631) 2024-11-11T23:54:08,091 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#C#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:08,091 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/d868a62c9e9649409a5d7afae0f75491 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d868a62c9e9649409a5d7afae0f75491 2024-11-11T23:54:08,092 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/a1f53ce4501648e5960911e38dc602b8 is 50, key is test_row_0/C:col10/1731369245273/Put/seqid=0 2024-11-11T23:54:08,098 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/A of d71f402ac6f3fd6fb0701033f3e67af7 into d868a62c9e9649409a5d7afae0f75491(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:08,099 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:08,099 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/A, priority=12, startTime=1731369247560; duration=0sec 2024-11-11T23:54:08,099 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:08,099 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:A 2024-11-11T23:54:08,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369248057, max timestamp=1731369248120Num uniques 859; 2024-11-11T23:54:08,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:08,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369248057, max timestamp=1731369248120Num uniques 859; 2024-11-11T23:54:08,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:08,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742000_1176 (size=12301) 2024-11-11T23:54:08,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369248057, max timestamp=1731369248120Num uniques 859; 2024-11-11T23:54:08,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:08,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742001_1177 (size=12677) 2024-11-11T23:54:08,160 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/a1f53ce4501648e5960911e38dc602b8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/a1f53ce4501648e5960911e38dc602b8 2024-11-11T23:54:08,165 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/C of d71f402ac6f3fd6fb0701033f3e67af7 into a1f53ce4501648e5960911e38dc602b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:08,165 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:08,165 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/C, priority=12, startTime=1731369247560; duration=0sec 2024-11-11T23:54:08,165 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:08,165 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:C 2024-11-11T23:54:08,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369248166, max timestamp=1731369248189Num uniques 1018; 2024-11-11T23:54:08,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369248166, max timestamp=1731369248189Num uniques 1018; 2024-11-11T23:54:08,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T23:54:08,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:54:08,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369248166, max timestamp=1731369248189Num uniques 1018; 2024-11-11T23:54:08,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:54:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T23:54:08,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369308227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369308230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369308345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369308345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,524 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/1ed615d58c2741ac9c9f8861d704cd6b 2024-11-11T23:54:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c44922e9b20042818bf0136b9428d633 is 50, key is test_row_0/C:col10/1731369245861/Put/seqid=0 2024-11-11T23:54:08,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742002_1178 (size=12301) 2024-11-11T23:54:08,541 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c44922e9b20042818bf0136b9428d633 2024-11-11T23:54:08,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/be963be5e469415183f7250edc0f82b9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/be963be5e469415183f7250edc0f82b9 2024-11-11T23:54:08,550 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/be963be5e469415183f7250edc0f82b9, entries=150, sequenceid=274, filesize=30.5 K 2024-11-11T23:54:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/1ed615d58c2741ac9c9f8861d704cd6b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1ed615d58c2741ac9c9f8861d704cd6b 2024-11-11T23:54:08,556 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1ed615d58c2741ac9c9f8861d704cd6b, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T23:54:08,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c44922e9b20042818bf0136b9428d633 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c44922e9b20042818bf0136b9428d633 2024-11-11T23:54:08,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c44922e9b20042818bf0136b9428d633, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T23:54:08,562 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for d71f402ac6f3fd6fb0701033f3e67af7 in 971ms, sequenceid=274, compaction requested=false 2024-11-11T23:54:08,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:08,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:08,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T23:54:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T23:54:08,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369248222, max timestamp=1731369248563Num uniques 1127; 2024-11-11T23:54:08,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-11T23:54:08,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369248222, max timestamp=1731369248563Num uniques 1027; 2024-11-11T23:54:08,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:54:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:08,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=71.56 KB heapSize=197.88 KB 2024-11-11T23:54:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:54:08,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T23:54:08,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5050 sec 2024-11-11T23:54:08,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c0b14532abb24972a104fb4a938cd2ff_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369248563/Put/seqid=0 2024-11-11T23:54:08,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.5200 sec 2024-11-11T23:54:08,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742003_1179 (size=12454) 2024-11-11T23:54:08,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369308624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369308628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369308734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369308734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369308945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:08,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369308954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:09,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:09,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c0b14532abb24972a104fb4a938cd2ff_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0b14532abb24972a104fb4a938cd2ff_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:09,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/fb4ac0b37b63416fb77c23fcf08f1af7, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:09,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/fb4ac0b37b63416fb77c23fcf08f1af7 is 175, key is test_row_0/A:col10/1731369248563/Put/seqid=0 2024-11-11T23:54:09,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742004_1180 (size=31255) 2024-11-11T23:54:09,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369309254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369309264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:09,425 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/fb4ac0b37b63416fb77c23fcf08f1af7 2024-11-11T23:54:09,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/efb6a3d215f5487eae98578cb249ba89 is 50, key is test_row_0/B:col10/1731369248563/Put/seqid=0 2024-11-11T23:54:09,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742005_1181 (size=12301) 2024-11-11T23:54:09,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/efb6a3d215f5487eae98578cb249ba89 2024-11-11T23:54:09,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c4489fdb5c694fdb9f6e060e390cf78e is 50, key is test_row_0/C:col10/1731369248563/Put/seqid=0 2024-11-11T23:54:09,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742006_1182 (size=12301) 2024-11-11T23:54:09,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369309514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:09,515 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369309764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:09,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369309785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:09,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:09,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:09,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c4489fdb5c694fdb9f6e060e390cf78e 2024-11-11T23:54:09,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/fb4ac0b37b63416fb77c23fcf08f1af7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/fb4ac0b37b63416fb77c23fcf08f1af7 2024-11-11T23:54:09,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/fb4ac0b37b63416fb77c23fcf08f1af7, entries=150, sequenceid=333, filesize=30.5 K 2024-11-11T23:54:09,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/efb6a3d215f5487eae98578cb249ba89 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/efb6a3d215f5487eae98578cb249ba89 2024-11-11T23:54:09,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/efb6a3d215f5487eae98578cb249ba89, entries=150, sequenceid=333, filesize=12.0 K 2024-11-11T23:54:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/c4489fdb5c694fdb9f6e060e390cf78e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c4489fdb5c694fdb9f6e060e390cf78e 2024-11-11T23:54:09,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c4489fdb5c694fdb9f6e060e390cf78e, entries=150, sequenceid=333, filesize=12.0 K 2024-11-11T23:54:09,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~197.83 KB/202576, currentSize=107.34 KB/109920 for d71f402ac6f3fd6fb0701033f3e67af7 in 1322ms, sequenceid=333, compaction requested=true 2024-11-11T23:54:09,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:09,891 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:09,891 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:09,892 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:09,892 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:09,892 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/B is initiating minor compaction (all files) 2024-11-11T23:54:09,892 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/A is initiating minor compaction (all files) 2024-11-11T23:54:09,892 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/A in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:09,892 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/B in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:09,892 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a31bfa55ca6a4feba9bd051f74d3dc88, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1ed615d58c2741ac9c9f8861d704cd6b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/efb6a3d215f5487eae98578cb249ba89] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=36.4 K 2024-11-11T23:54:09,892 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d868a62c9e9649409a5d7afae0f75491, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/be963be5e469415183f7250edc0f82b9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/fb4ac0b37b63416fb77c23fcf08f1af7] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=91.9 K 2024-11-11T23:54:09,892 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:09,892 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d868a62c9e9649409a5d7afae0f75491, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/be963be5e469415183f7250edc0f82b9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/fb4ac0b37b63416fb77c23fcf08f1af7] 2024-11-11T23:54:09,893 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d868a62c9e9649409a5d7afae0f75491, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369245273 2024-11-11T23:54:09,893 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a31bfa55ca6a4feba9bd051f74d3dc88, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369245273 2024-11-11T23:54:09,893 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ed615d58c2741ac9c9f8861d704cd6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731369245861 2024-11-11T23:54:09,893 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting be963be5e469415183f7250edc0f82b9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731369245861 2024-11-11T23:54:09,894 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting efb6a3d215f5487eae98578cb249ba89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731369248225 2024-11-11T23:54:09,894 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb4ac0b37b63416fb77c23fcf08f1af7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731369248225 2024-11-11T23:54:09,901 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:09,906 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411116ad713051eb04b999f1559af469afbc4_d71f402ac6f3fd6fb0701033f3e67af7 store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:09,907 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:09,908 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411116ad713051eb04b999f1559af469afbc4_d71f402ac6f3fd6fb0701033f3e67af7, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:09,908 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/118da089e65f47709a9b883266587e38 is 50, key is test_row_0/B:col10/1731369248563/Put/seqid=0 2024-11-11T23:54:09,908 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116ad713051eb04b999f1559af469afbc4_d71f402ac6f3fd6fb0701033f3e67af7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:09,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742008_1184 (size=12779) 2024-11-11T23:54:09,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742007_1183 (size=4469) 2024-11-11T23:54:09,915 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#A#compaction#153 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:09,915 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/7722bb4d88274888a31ab6cc1075e424 is 175, key is test_row_0/A:col10/1731369248563/Put/seqid=0 2024-11-11T23:54:09,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742009_1185 (size=31733) 2024-11-11T23:54:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T23:54:10,214 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:10,217 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T23:54:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T23:54:10,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:10,221 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:10,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:10,320 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/118da089e65f47709a9b883266587e38 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/118da089e65f47709a9b883266587e38 2024-11-11T23:54:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T23:54:10,325 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/7722bb4d88274888a31ab6cc1075e424 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7722bb4d88274888a31ab6cc1075e424 2024-11-11T23:54:10,328 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/B of d71f402ac6f3fd6fb0701033f3e67af7 into 118da089e65f47709a9b883266587e38(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:10,328 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:10,328 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/B, priority=13, startTime=1731369249891; duration=0sec 2024-11-11T23:54:10,329 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:10,329 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:B 2024-11-11T23:54:10,329 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:10,333 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:10,333 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/C is initiating minor compaction (all files) 2024-11-11T23:54:10,333 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/C in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:10,333 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/a1f53ce4501648e5960911e38dc602b8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c44922e9b20042818bf0136b9428d633, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c4489fdb5c694fdb9f6e060e390cf78e] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=36.4 K 2024-11-11T23:54:10,335 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/A of d71f402ac6f3fd6fb0701033f3e67af7 into 7722bb4d88274888a31ab6cc1075e424(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:10,335 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:10,335 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/A, priority=13, startTime=1731369249891; duration=0sec 2024-11-11T23:54:10,335 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:10,335 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:A 2024-11-11T23:54:10,335 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f53ce4501648e5960911e38dc602b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369245273 2024-11-11T23:54:10,336 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c44922e9b20042818bf0136b9428d633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731369245861 2024-11-11T23:54:10,338 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c4489fdb5c694fdb9f6e060e390cf78e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731369248225 2024-11-11T23:54:10,350 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:10,351 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/19c051ef42d74357895419acdecc8e20 is 50, key is test_row_0/C:col10/1731369248563/Put/seqid=0 2024-11-11T23:54:10,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742010_1186 (size=12779) 2024-11-11T23:54:10,373 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/19c051ef42d74357895419acdecc8e20 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/19c051ef42d74357895419acdecc8e20 2024-11-11T23:54:10,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T23:54:10,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:10,376 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T23:54:10,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:10,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:10,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:10,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:10,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:10,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:10,380 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/C of d71f402ac6f3fd6fb0701033f3e67af7 into 19c051ef42d74357895419acdecc8e20(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:10,380 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:10,380 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/C, priority=13, startTime=1731369249891; duration=0sec 2024-11-11T23:54:10,380 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:10,380 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:C 2024-11-11T23:54:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111135af3696169c4c34a646f0cbdba9251d_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369248612/Put/seqid=0 2024-11-11T23:54:10,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742011_1187 (size=12454) 2024-11-11T23:54:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T23:54:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:10,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:10,806 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111135af3696169c4c34a646f0cbdba9251d_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111135af3696169c4c34a646f0cbdba9251d_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:10,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:10,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/7438755132db48098e1d0e69ef6e52ba, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369310806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:10,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369310808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:10,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/7438755132db48098e1d0e69ef6e52ba is 175, key is test_row_0/A:col10/1731369248612/Put/seqid=0 2024-11-11T23:54:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:10,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742012_1188 (size=31255) 2024-11-11T23:54:10,817 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/7438755132db48098e1d0e69ef6e52ba 2024-11-11T23:54:10,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/c0a401d9f266447dad342d5da8d232e4 is 50, key is test_row_0/B:col10/1731369248612/Put/seqid=0 2024-11-11T23:54:10,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742013_1189 (size=12301) 2024-11-11T23:54:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T23:54:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369310914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:10,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369310915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:11,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369311134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:11,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:11,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369311134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:11,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:11,230 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/c0a401d9f266447dad342d5da8d232e4 2024-11-11T23:54:11,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/324124cab9684f4495901ab9643c5a49 is 50, key is test_row_0/C:col10/1731369248612/Put/seqid=0 2024-11-11T23:54:11,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742014_1190 (size=12301) 2024-11-11T23:54:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T23:54:11,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369311445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369311445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:11,643 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/324124cab9684f4495901ab9643c5a49 2024-11-11T23:54:11,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/7438755132db48098e1d0e69ef6e52ba as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7438755132db48098e1d0e69ef6e52ba 2024-11-11T23:54:11,652 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7438755132db48098e1d0e69ef6e52ba, entries=150, sequenceid=354, filesize=30.5 K 2024-11-11T23:54:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/c0a401d9f266447dad342d5da8d232e4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c0a401d9f266447dad342d5da8d232e4 2024-11-11T23:54:11,657 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c0a401d9f266447dad342d5da8d232e4, entries=150, sequenceid=354, filesize=12.0 K 2024-11-11T23:54:11,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/324124cab9684f4495901ab9643c5a49 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/324124cab9684f4495901ab9643c5a49 2024-11-11T23:54:11,663 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/324124cab9684f4495901ab9643c5a49, entries=150, sequenceid=354, filesize=12.0 K 2024-11-11T23:54:11,664 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d71f402ac6f3fd6fb0701033f3e67af7 in 1288ms, sequenceid=354, compaction requested=false 2024-11-11T23:54:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T23:54:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T23:54:11,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T23:54:11,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4440 sec 2024-11-11T23:54:11,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4530 sec 2024-11-11T23:54:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:11,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T23:54:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:11,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111184dc67d1d8a24884a64a0eb4a76001a4_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369251956/Put/seqid=0 2024-11-11T23:54:11,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742015_1191 (size=14994) 2024-11-11T23:54:11,986 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:11,989 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111184dc67d1d8a24884a64a0eb4a76001a4_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111184dc67d1d8a24884a64a0eb4a76001a4_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:11,990 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/afeb67e598464cf691203be354a25138, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:11,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/afeb67e598464cf691203be354a25138 is 175, key is test_row_0/A:col10/1731369251956/Put/seqid=0 2024-11-11T23:54:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369311985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369311987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:11,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742016_1192 (size=39949) 2024-11-11T23:54:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:12,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369312104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:12,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369312104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:12,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369312315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369312315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T23:54:12,365 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:12,365 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T23:54:12,367 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T23:54:12,367 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:12,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:12,396 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/afeb67e598464cf691203be354a25138 2024-11-11T23:54:12,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/376b3061dfb14b6ca8a2005d27e25c7a is 50, key is test_row_0/B:col10/1731369251956/Put/seqid=0 2024-11-11T23:54:12,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742017_1193 (size=12301) 2024-11-11T23:54:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T23:54:12,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T23:54:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:12,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369312624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369312625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:12,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T23:54:12,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:12,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:12,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:12,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T23:54:12,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/376b3061dfb14b6ca8a2005d27e25c7a 2024-11-11T23:54:12,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/5bd9600232934d9190c15346fe59b8ad is 50, key is test_row_0/C:col10/1731369251956/Put/seqid=0 2024-11-11T23:54:12,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742018_1194 (size=12301) 2024-11-11T23:54:12,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T23:54:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:12,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T23:54:12,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:12,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:12,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:12,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T23:54:13,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T23:54:13,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:13,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:13,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:13,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:13,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:13,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:13,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:13,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369313144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369313144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:13,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/5bd9600232934d9190c15346fe59b8ad 2024-11-11T23:54:13,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/afeb67e598464cf691203be354a25138 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/afeb67e598464cf691203be354a25138 2024-11-11T23:54:13,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/afeb67e598464cf691203be354a25138, entries=200, sequenceid=369, filesize=39.0 K 2024-11-11T23:54:13,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/376b3061dfb14b6ca8a2005d27e25c7a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/376b3061dfb14b6ca8a2005d27e25c7a 2024-11-11T23:54:13,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/376b3061dfb14b6ca8a2005d27e25c7a, entries=150, sequenceid=369, filesize=12.0 K 2024-11-11T23:54:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/5bd9600232934d9190c15346fe59b8ad as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/5bd9600232934d9190c15346fe59b8ad 2024-11-11T23:54:13,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/5bd9600232934d9190c15346fe59b8ad, entries=150, sequenceid=369, filesize=12.0 K 2024-11-11T23:54:13,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d71f402ac6f3fd6fb0701033f3e67af7 in 1286ms, sequenceid=369, compaction requested=true 2024-11-11T23:54:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:13,243 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:13,243 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:13,244 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:13,244 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:13,244 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/B is initiating minor compaction (all files) 2024-11-11T23:54:13,244 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/A is initiating minor compaction (all files) 2024-11-11T23:54:13,244 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/B in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:13,244 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/A in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:13,244 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7722bb4d88274888a31ab6cc1075e424, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7438755132db48098e1d0e69ef6e52ba, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/afeb67e598464cf691203be354a25138] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=100.5 K 2024-11-11T23:54:13,244 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/118da089e65f47709a9b883266587e38, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c0a401d9f266447dad342d5da8d232e4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/376b3061dfb14b6ca8a2005d27e25c7a] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=36.5 K 2024-11-11T23:54:13,244 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:13,244 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7722bb4d88274888a31ab6cc1075e424, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7438755132db48098e1d0e69ef6e52ba, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/afeb67e598464cf691203be354a25138] 2024-11-11T23:54:13,245 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 118da089e65f47709a9b883266587e38, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731369248225 2024-11-11T23:54:13,245 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7722bb4d88274888a31ab6cc1075e424, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731369248225 2024-11-11T23:54:13,245 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c0a401d9f266447dad342d5da8d232e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731369248612 2024-11-11T23:54:13,245 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7438755132db48098e1d0e69ef6e52ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731369248612 2024-11-11T23:54:13,245 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 376b3061dfb14b6ca8a2005d27e25c7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731369250804 2024-11-11T23:54:13,245 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting afeb67e598464cf691203be354a25138, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731369250796 2024-11-11T23:54:13,253 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:13,255 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:13,256 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/451d9609efd542599de8cd458c4bee1a is 50, key is test_row_0/B:col10/1731369251956/Put/seqid=0 2024-11-11T23:54:13,263 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111107e0ed2bf18e46e488e0ac6d91f6ed97_d71f402ac6f3fd6fb0701033f3e67af7 store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:13,265 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111107e0ed2bf18e46e488e0ac6d91f6ed97_d71f402ac6f3fd6fb0701033f3e67af7, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:13,266 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111107e0ed2bf18e46e488e0ac6d91f6ed97_d71f402ac6f3fd6fb0701033f3e67af7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:13,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742019_1195 (size=12881) 2024-11-11T23:54:13,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742020_1196 (size=4469) 2024-11-11T23:54:13,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T23:54:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:13,285 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T23:54:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:13,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d939a5138eed4340bf4698a3e5b22092_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369251985/Put/seqid=0 2024-11-11T23:54:13,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742021_1197 (size=12454) 2024-11-11T23:54:13,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:13,308 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d939a5138eed4340bf4698a3e5b22092_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d939a5138eed4340bf4698a3e5b22092_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:13,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/232cf7640f584545bf4c89acfc6ab52d, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:13,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/232cf7640f584545bf4c89acfc6ab52d is 175, key is test_row_0/A:col10/1731369251985/Put/seqid=0 2024-11-11T23:54:13,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742022_1198 (size=31255) 2024-11-11T23:54:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T23:54:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:13,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:13,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369313567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:13,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369313674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:13,679 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/451d9609efd542599de8cd458c4bee1a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/451d9609efd542599de8cd458c4bee1a 2024-11-11T23:54:13,680 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:13,681 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/215f8fcf2f734010a871312902430e66 is 175, key is test_row_0/A:col10/1731369251956/Put/seqid=0 2024-11-11T23:54:13,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742023_1199 (size=31835) 2024-11-11T23:54:13,685 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/B of d71f402ac6f3fd6fb0701033f3e67af7 into 451d9609efd542599de8cd458c4bee1a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:13,685 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:13,685 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/B, priority=13, startTime=1731369253243; duration=0sec 2024-11-11T23:54:13,685 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:13,685 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:B 2024-11-11T23:54:13,685 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:13,686 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:13,686 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/C is initiating minor compaction (all files) 2024-11-11T23:54:13,686 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/C in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:13,686 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/19c051ef42d74357895419acdecc8e20, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/324124cab9684f4495901ab9643c5a49, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/5bd9600232934d9190c15346fe59b8ad] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=36.5 K 2024-11-11T23:54:13,687 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c051ef42d74357895419acdecc8e20, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731369248225 2024-11-11T23:54:13,687 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 324124cab9684f4495901ab9643c5a49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731369248612 2024-11-11T23:54:13,688 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd9600232934d9190c15346fe59b8ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731369250804 2024-11-11T23:54:13,694 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:13,694 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/b5c6b056d6a843bfaf733e8099eafbda is 50, key is test_row_0/C:col10/1731369251956/Put/seqid=0 2024-11-11T23:54:13,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742024_1200 (size=12881) 2024-11-11T23:54:13,726 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/232cf7640f584545bf4c89acfc6ab52d 2024-11-11T23:54:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/2e764d6b7e634199854c385dbb4ca0ee is 50, key is test_row_0/B:col10/1731369251985/Put/seqid=0 2024-11-11T23:54:13,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742025_1201 (size=12301) 2024-11-11T23:54:13,739 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/2e764d6b7e634199854c385dbb4ca0ee 2024-11-11T23:54:13,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/56ed9ce214d34fc6b2264c5590c2b6c7 is 50, key is test_row_0/C:col10/1731369251985/Put/seqid=0 2024-11-11T23:54:13,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742026_1202 (size=12301) 2024-11-11T23:54:13,754 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/56ed9ce214d34fc6b2264c5590c2b6c7 2024-11-11T23:54:13,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/232cf7640f584545bf4c89acfc6ab52d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/232cf7640f584545bf4c89acfc6ab52d 2024-11-11T23:54:13,772 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/232cf7640f584545bf4c89acfc6ab52d, entries=150, sequenceid=387, filesize=30.5 K 2024-11-11T23:54:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/2e764d6b7e634199854c385dbb4ca0ee as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/2e764d6b7e634199854c385dbb4ca0ee 2024-11-11T23:54:13,779 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/2e764d6b7e634199854c385dbb4ca0ee, entries=150, sequenceid=387, filesize=12.0 K 2024-11-11T23:54:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/56ed9ce214d34fc6b2264c5590c2b6c7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/56ed9ce214d34fc6b2264c5590c2b6c7 2024-11-11T23:54:13,797 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/56ed9ce214d34fc6b2264c5590c2b6c7, entries=150, sequenceid=387, filesize=12.0 K 2024-11-11T23:54:13,797 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d71f402ac6f3fd6fb0701033f3e67af7 in 512ms, sequenceid=387, compaction requested=false 2024-11-11T23:54:13,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:13,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:13,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T23:54:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T23:54:13,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T23:54:13,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-11T23:54:13,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4400 sec 2024-11-11T23:54:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:13,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T23:54:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:13,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111103da499855504d918d1cc42e18309c4c_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369253886/Put/seqid=0 2024-11-11T23:54:13,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742027_1203 (size=14994) 2024-11-11T23:54:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369313953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369313954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:13,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43316 deadline: 1731369313994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:13,999 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369314065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369314074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,090 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/215f8fcf2f734010a871312902430e66 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/215f8fcf2f734010a871312902430e66 2024-11-11T23:54:14,095 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/A of d71f402ac6f3fd6fb0701033f3e67af7 into 215f8fcf2f734010a871312902430e66(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:14,095 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:14,095 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/A, priority=13, startTime=1731369253243; duration=0sec 2024-11-11T23:54:14,095 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:14,095 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:A 2024-11-11T23:54:14,103 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/b5c6b056d6a843bfaf733e8099eafbda as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/b5c6b056d6a843bfaf733e8099eafbda 2024-11-11T23:54:14,109 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/C of d71f402ac6f3fd6fb0701033f3e67af7 into b5c6b056d6a843bfaf733e8099eafbda(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:14,109 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:14,109 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/C, priority=13, startTime=1731369253243; duration=0sec 2024-11-11T23:54:14,109 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:14,109 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:C 2024-11-11T23:54:14,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43344 deadline: 1731369314155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:14,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43294 deadline: 1731369314155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:14,156 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:14,156 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in d71f402ac6f3fd6fb0701033f3e67af7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369314274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369314284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,301 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:14,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111103da499855504d918d1cc42e18309c4c_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111103da499855504d918d1cc42e18309c4c_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:14,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/dc8580eee7764c138c0f643fbc67168b, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:14,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/dc8580eee7764c138c0f643fbc67168b is 175, key is test_row_0/A:col10/1731369253886/Put/seqid=0 2024-11-11T23:54:14,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742028_1204 (size=39949) 2024-11-11T23:54:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T23:54:14,514 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:14,515 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T23:54:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T23:54:14,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:14,517 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:14,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:14,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369314585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369314595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T23:54:14,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T23:54:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:14,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:14,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/dc8580eee7764c138c0f643fbc67168b 2024-11-11T23:54:14,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/4e6c240c29584f5bbf7c2d976ea95642 is 50, key is test_row_0/B:col10/1731369253886/Put/seqid=0 2024-11-11T23:54:14,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742029_1205 (size=12301) 2024-11-11T23:54:14,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T23:54:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:14,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T23:54:14,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T23:54:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:14,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43292 deadline: 1731369315094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:15,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43314 deadline: 1731369315104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:15,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/4e6c240c29584f5bbf7c2d976ea95642 2024-11-11T23:54:15,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T23:54:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:15,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:15,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/94b37ffa97394a3d92ba0af9f17fe4f4 is 50, key is test_row_0/C:col10/1731369253886/Put/seqid=0 2024-11-11T23:54:15,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742030_1206 (size=12301) 2024-11-11T23:54:15,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/94b37ffa97394a3d92ba0af9f17fe4f4 2024-11-11T23:54:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T23:54:15,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/dc8580eee7764c138c0f643fbc67168b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/dc8580eee7764c138c0f643fbc67168b 2024-11-11T23:54:15,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/dc8580eee7764c138c0f643fbc67168b, entries=200, sequenceid=403, filesize=39.0 K 2024-11-11T23:54:15,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/4e6c240c29584f5bbf7c2d976ea95642 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4e6c240c29584f5bbf7c2d976ea95642 2024-11-11T23:54:15,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4e6c240c29584f5bbf7c2d976ea95642, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T23:54:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/94b37ffa97394a3d92ba0af9f17fe4f4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/94b37ffa97394a3d92ba0af9f17fe4f4 2024-11-11T23:54:15,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/94b37ffa97394a3d92ba0af9f17fe4f4, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T23:54:15,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d71f402ac6f3fd6fb0701033f3e67af7 in 1281ms, sequenceid=403, compaction requested=true 2024-11-11T23:54:15,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:15,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:15,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:15,168 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:15,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:15,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:15,168 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:15,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d71f402ac6f3fd6fb0701033f3e67af7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:15,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:15,170 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:15,170 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:15,170 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/B is initiating minor compaction (all files) 2024-11-11T23:54:15,170 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/A is initiating minor compaction (all files) 2024-11-11T23:54:15,170 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/A in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:15,170 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/B in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:15,170 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/451d9609efd542599de8cd458c4bee1a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/2e764d6b7e634199854c385dbb4ca0ee, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4e6c240c29584f5bbf7c2d976ea95642] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=36.6 K 2024-11-11T23:54:15,170 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/215f8fcf2f734010a871312902430e66, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/232cf7640f584545bf4c89acfc6ab52d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/dc8580eee7764c138c0f643fbc67168b] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=100.6 K 2024-11-11T23:54:15,170 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:15,170 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/215f8fcf2f734010a871312902430e66, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/232cf7640f584545bf4c89acfc6ab52d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/dc8580eee7764c138c0f643fbc67168b] 2024-11-11T23:54:15,170 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 451d9609efd542599de8cd458c4bee1a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731369250804 2024-11-11T23:54:15,171 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 215f8fcf2f734010a871312902430e66, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731369250804 2024-11-11T23:54:15,171 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e764d6b7e634199854c385dbb4ca0ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731369251975 2024-11-11T23:54:15,171 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 232cf7640f584545bf4c89acfc6ab52d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731369251975 2024-11-11T23:54:15,171 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6c240c29584f5bbf7c2d976ea95642, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731369253554 2024-11-11T23:54:15,172 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc8580eee7764c138c0f643fbc67168b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731369253554 2024-11-11T23:54:15,181 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:15,182 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/c109ae319fe9401ab4148aa01d306f69 is 50, key is test_row_0/B:col10/1731369253886/Put/seqid=0 2024-11-11T23:54:15,186 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:15,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742031_1207 (size=12983) 2024-11-11T23:54:15,196 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111beca58485a6e4c659165f1711d61234a_d71f402ac6f3fd6fb0701033f3e67af7 store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:15,198 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111beca58485a6e4c659165f1711d61234a_d71f402ac6f3fd6fb0701033f3e67af7, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:15,198 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111beca58485a6e4c659165f1711d61234a_d71f402ac6f3fd6fb0701033f3e67af7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:15,210 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/c109ae319fe9401ab4148aa01d306f69 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c109ae319fe9401ab4148aa01d306f69 2024-11-11T23:54:15,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742032_1208 (size=4469) 2024-11-11T23:54:15,222 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/B of d71f402ac6f3fd6fb0701033f3e67af7 into c109ae319fe9401ab4148aa01d306f69(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:15,222 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:15,222 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/B, priority=13, startTime=1731369255168; duration=0sec 2024-11-11T23:54:15,222 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:15,222 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:B 2024-11-11T23:54:15,222 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:15,227 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:15,227 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): d71f402ac6f3fd6fb0701033f3e67af7/C is initiating minor compaction (all files) 2024-11-11T23:54:15,227 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d71f402ac6f3fd6fb0701033f3e67af7/C in TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:15,227 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/b5c6b056d6a843bfaf733e8099eafbda, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/56ed9ce214d34fc6b2264c5590c2b6c7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/94b37ffa97394a3d92ba0af9f17fe4f4] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp, totalSize=36.6 K 2024-11-11T23:54:15,227 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b5c6b056d6a843bfaf733e8099eafbda, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731369250804 2024-11-11T23:54:15,228 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ed9ce214d34fc6b2264c5590c2b6c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731369251975 2024-11-11T23:54:15,229 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b37ffa97394a3d92ba0af9f17fe4f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731369253554 2024-11-11T23:54:15,240 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:15,240 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/3f49e01b69bb42ed89b1c7eaaef187f5 is 50, key is test_row_0/C:col10/1731369253886/Put/seqid=0 2024-11-11T23:54:15,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742033_1209 (size=12983) 2024-11-11T23:54:15,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T23:54:15,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:15,284 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T23:54:15,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:15,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:15,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:15,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:15,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:15,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:15,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111166f1ad31af9c419182bc1f8c2b9e7d81_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369253952/Put/seqid=0 2024-11-11T23:54:15,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742034_1210 (size=12454) 2024-11-11T23:54:15,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:15,303 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111166f1ad31af9c419182bc1f8c2b9e7d81_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166f1ad31af9c419182bc1f8c2b9e7d81_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/b1cdc3b9acd44b9cb37d726433e620fa, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/b1cdc3b9acd44b9cb37d726433e620fa is 175, key is test_row_0/A:col10/1731369253952/Put/seqid=0 2024-11-11T23:54:15,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742035_1211 (size=31255) 2024-11-11T23:54:15,337 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/b1cdc3b9acd44b9cb37d726433e620fa 2024-11-11T23:54:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/671b1ecc8fba452a99f37c9032933c23 is 50, key is test_row_0/B:col10/1731369253952/Put/seqid=0 2024-11-11T23:54:15,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742036_1212 (size=12301) 2024-11-11T23:54:15,574 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-11T23:54:15,574 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:15,574 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:15,575 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:15,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:15,575 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-11T23:54:15,575 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:15,575 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:15,575 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:15,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:15,576 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-11T23:54:15,576 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:15,576 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:15,577 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:15,577 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:15,578 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-11T23:54:15,578 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:15,578 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:15,578 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:15,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:15,625 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d71f402ac6f3fd6fb0701033f3e67af7#A#compaction#172 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:15,625 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/4a1b1224721b4fa7b35cc584e19a1e67 is 175, key is test_row_0/A:col10/1731369253886/Put/seqid=0 2024-11-11T23:54:15,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742037_1213 (size=31937) 2024-11-11T23:54:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T23:54:15,673 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/3f49e01b69bb42ed89b1c7eaaef187f5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/3f49e01b69bb42ed89b1c7eaaef187f5 2024-11-11T23:54:15,679 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/C of d71f402ac6f3fd6fb0701033f3e67af7 into 3f49e01b69bb42ed89b1c7eaaef187f5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:15,679 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:15,679 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/C, priority=13, startTime=1731369255168; duration=0sec 2024-11-11T23:54:15,679 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:15,679 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:C 2024-11-11T23:54:15,764 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/671b1ecc8fba452a99f37c9032933c23 2024-11-11T23:54:15,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/4a8e21f0269f457eb1aceac9b096d3ce is 50, key is test_row_0/C:col10/1731369253952/Put/seqid=0 2024-11-11T23:54:15,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742038_1214 (size=12301) 2024-11-11T23:54:16,040 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/4a1b1224721b4fa7b35cc584e19a1e67 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/4a1b1224721b4fa7b35cc584e19a1e67 2024-11-11T23:54:16,048 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d71f402ac6f3fd6fb0701033f3e67af7/A of d71f402ac6f3fd6fb0701033f3e67af7 into 4a1b1224721b4fa7b35cc584e19a1e67(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:16,048 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:16,048 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., storeName=d71f402ac6f3fd6fb0701033f3e67af7/A, priority=13, startTime=1731369255168; duration=0sec 2024-11-11T23:54:16,048 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:16,048 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d71f402ac6f3fd6fb0701033f3e67af7:A 2024-11-11T23:54:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:16,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. as already flushing 2024-11-11T23:54:16,106 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-11T23:54:16,106 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:16,106 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:16,106 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:16,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:16,116 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-11-11T23:54:16,116 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:16,116 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:16,116 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:16,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:16,168 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-11-11T23:54:16,168 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-11T23:54:16,168 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:16,168 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:16,168 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:16,168 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:16,169 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:16,169 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:16,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:16,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:16,184 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/4a8e21f0269f457eb1aceac9b096d3ce 2024-11-11T23:54:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/b1cdc3b9acd44b9cb37d726433e620fa as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/b1cdc3b9acd44b9cb37d726433e620fa 2024-11-11T23:54:16,198 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/b1cdc3b9acd44b9cb37d726433e620fa, entries=150, sequenceid=423, filesize=30.5 K 2024-11-11T23:54:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/671b1ecc8fba452a99f37c9032933c23 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/671b1ecc8fba452a99f37c9032933c23 2024-11-11T23:54:16,203 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/671b1ecc8fba452a99f37c9032933c23, entries=150, sequenceid=423, filesize=12.0 K 2024-11-11T23:54:16,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/4a8e21f0269f457eb1aceac9b096d3ce as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/4a8e21f0269f457eb1aceac9b096d3ce 2024-11-11T23:54:16,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/4a8e21f0269f457eb1aceac9b096d3ce, entries=150, sequenceid=423, filesize=12.0 K 2024-11-11T23:54:16,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for d71f402ac6f3fd6fb0701033f3e67af7 in 924ms, sequenceid=423, compaction requested=false 2024-11-11T23:54:16,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for d71f402ac6f3fd6fb0701033f3e67af7: 2024-11-11T23:54:16,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:16,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T23:54:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T23:54:16,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T23:54:16,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6920 sec 2024-11-11T23:54:16,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6960 sec 2024-11-11T23:54:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T23:54:16,665 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:19,731 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:54:24,027 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-11T23:54:24,027 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:24,027 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:24,027 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:24,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2822 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2789 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3650 rows 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1213 2024-11-11T23:54:24,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3638 rows 2024-11-11T23:54:24,028 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:54:24,028 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:54:24,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:24,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:24,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:24,029 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T23:54:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T23:54:24,032 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369264032"}]},"ts":"1731369264032"} 2024-11-11T23:54:24,034 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T23:54:24,034 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T23:54:24,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:54:24,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, UNASSIGN}] 2024-11-11T23:54:24,037 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, UNASSIGN 2024-11-11T23:54:24,037 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=d71f402ac6f3fd6fb0701033f3e67af7, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:24,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, UNASSIGN because future has completed 2024-11-11T23:54:24,039 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:54:24,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:54:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T23:54:24,191 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:24,191 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:54:24,191 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing d71f402ac6f3fd6fb0701033f3e67af7, disabling compactions & flushes 2024-11-11T23:54:24,191 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:24,191 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:24,191 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. after waiting 0 ms 2024-11-11T23:54:24,191 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:24,191 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing d71f402ac6f3fd6fb0701033f3e67af7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T23:54:24,192 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=A 2024-11-11T23:54:24,192 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:24,192 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=B 2024-11-11T23:54:24,192 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:24,192 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d71f402ac6f3fd6fb0701033f3e67af7, store=C 2024-11-11T23:54:24,192 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:24,198 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111726256e007cf4ffbb9ee644603348a53_d71f402ac6f3fd6fb0701033f3e67af7 is 50, key is test_row_0/A:col10/1731369256105/Put/seqid=0 2024-11-11T23:54:24,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742039_1215 (size=12454) 2024-11-11T23:54:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T23:54:24,602 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:24,608 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111726256e007cf4ffbb9ee644603348a53_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111726256e007cf4ffbb9ee644603348a53_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:24,609 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/13ef7669b8024ef9a59555849a225c08, store: [table=TestAcidGuarantees family=A region=d71f402ac6f3fd6fb0701033f3e67af7] 2024-11-11T23:54:24,610 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/13ef7669b8024ef9a59555849a225c08 is 175, key is test_row_0/A:col10/1731369256105/Put/seqid=0 2024-11-11T23:54:24,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742040_1216 (size=31255) 2024-11-11T23:54:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T23:54:25,016 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/13ef7669b8024ef9a59555849a225c08 2024-11-11T23:54:25,028 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/1de27900d33843d583e22b8463b64e4e is 50, key is test_row_0/B:col10/1731369256105/Put/seqid=0 2024-11-11T23:54:25,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742041_1217 (size=12301) 2024-11-11T23:54:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T23:54:25,432 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/1de27900d33843d583e22b8463b64e4e 2024-11-11T23:54:25,443 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/e8ccb3464b474777a2a24ba3da7dfdf8 is 50, key is test_row_0/C:col10/1731369256105/Put/seqid=0 2024-11-11T23:54:25,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742042_1218 (size=12301) 2024-11-11T23:54:25,849 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/e8ccb3464b474777a2a24ba3da7dfdf8 2024-11-11T23:54:25,857 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/A/13ef7669b8024ef9a59555849a225c08 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/13ef7669b8024ef9a59555849a225c08 2024-11-11T23:54:25,864 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/13ef7669b8024ef9a59555849a225c08, entries=150, sequenceid=433, filesize=30.5 K 2024-11-11T23:54:25,865 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/B/1de27900d33843d583e22b8463b64e4e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1de27900d33843d583e22b8463b64e4e 2024-11-11T23:54:25,870 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1de27900d33843d583e22b8463b64e4e, entries=150, sequenceid=433, filesize=12.0 K 2024-11-11T23:54:25,871 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/.tmp/C/e8ccb3464b474777a2a24ba3da7dfdf8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/e8ccb3464b474777a2a24ba3da7dfdf8 2024-11-11T23:54:25,875 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/e8ccb3464b474777a2a24ba3da7dfdf8, entries=150, sequenceid=433, filesize=12.0 K 2024-11-11T23:54:25,876 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d71f402ac6f3fd6fb0701033f3e67af7 in 1685ms, sequenceid=433, compaction requested=true 2024-11-11T23:54:25,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3793848cf1da4a65992b64dcfe33d195, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/01563991f87c475f9f03ad274461770b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/0b12338794f745a096350808222f76ed, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e2370ebb7dbf4ffba18e3ade07e77a8d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/610d3387f1b847929084b5adc835ff19, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d5a71d6c40be4437bd6a6497c61febe7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/2e32a05fe1bd41b6bc554ff08008bd87, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3d947a462580496eb17a583aa3cd85b5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/c997f60c0c1f41979bec817b80dac9c7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/8e42eedba3dc45bf89e6587b13a15af2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e280818aa2bf4a06a04af320fd5dcd1f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d868a62c9e9649409a5d7afae0f75491, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/be963be5e469415183f7250edc0f82b9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7722bb4d88274888a31ab6cc1075e424, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/fb4ac0b37b63416fb77c23fcf08f1af7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7438755132db48098e1d0e69ef6e52ba, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/afeb67e598464cf691203be354a25138, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/215f8fcf2f734010a871312902430e66, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/232cf7640f584545bf4c89acfc6ab52d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/dc8580eee7764c138c0f643fbc67168b] to archive 2024-11-11T23:54:25,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:54:25,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3793848cf1da4a65992b64dcfe33d195 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3793848cf1da4a65992b64dcfe33d195 2024-11-11T23:54:25,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/01563991f87c475f9f03ad274461770b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/01563991f87c475f9f03ad274461770b 2024-11-11T23:54:25,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/0b12338794f745a096350808222f76ed to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/0b12338794f745a096350808222f76ed 2024-11-11T23:54:25,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e2370ebb7dbf4ffba18e3ade07e77a8d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e2370ebb7dbf4ffba18e3ade07e77a8d 2024-11-11T23:54:25,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/610d3387f1b847929084b5adc835ff19 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/610d3387f1b847929084b5adc835ff19 2024-11-11T23:54:25,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d5a71d6c40be4437bd6a6497c61febe7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d5a71d6c40be4437bd6a6497c61febe7 2024-11-11T23:54:25,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/2e32a05fe1bd41b6bc554ff08008bd87 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/2e32a05fe1bd41b6bc554ff08008bd87 2024-11-11T23:54:25,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3d947a462580496eb17a583aa3cd85b5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/3d947a462580496eb17a583aa3cd85b5 2024-11-11T23:54:25,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/c997f60c0c1f41979bec817b80dac9c7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/c997f60c0c1f41979bec817b80dac9c7 2024-11-11T23:54:25,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/8e42eedba3dc45bf89e6587b13a15af2 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/8e42eedba3dc45bf89e6587b13a15af2 2024-11-11T23:54:25,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e280818aa2bf4a06a04af320fd5dcd1f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/e280818aa2bf4a06a04af320fd5dcd1f 2024-11-11T23:54:25,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d868a62c9e9649409a5d7afae0f75491 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/d868a62c9e9649409a5d7afae0f75491 2024-11-11T23:54:25,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/be963be5e469415183f7250edc0f82b9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/be963be5e469415183f7250edc0f82b9 2024-11-11T23:54:25,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7722bb4d88274888a31ab6cc1075e424 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7722bb4d88274888a31ab6cc1075e424 2024-11-11T23:54:25,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/fb4ac0b37b63416fb77c23fcf08f1af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/fb4ac0b37b63416fb77c23fcf08f1af7 2024-11-11T23:54:25,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7438755132db48098e1d0e69ef6e52ba to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/7438755132db48098e1d0e69ef6e52ba 2024-11-11T23:54:25,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/afeb67e598464cf691203be354a25138 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/afeb67e598464cf691203be354a25138 2024-11-11T23:54:25,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/215f8fcf2f734010a871312902430e66 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/215f8fcf2f734010a871312902430e66 2024-11-11T23:54:25,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/232cf7640f584545bf4c89acfc6ab52d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/232cf7640f584545bf4c89acfc6ab52d 2024-11-11T23:54:25,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/dc8580eee7764c138c0f643fbc67168b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/dc8580eee7764c138c0f643fbc67168b 2024-11-11T23:54:25,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/6207fc4b3b0a4baaa597573de927436d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/dc415fdc23cd4c8caf08d6a865e03a4c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/90b920c6e64c469e920378db69a9721c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/43fb5f379335446e8132ebbbb75c4ddd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4ab6229a75114ef695be4a8adc0fc9be, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/70bb24c8c53f41f383a258561c164476, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/ca54f03aeaf141fab0892a1dbf33ef0f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/572497a1c8bd48249e80cd8fec1bdcb4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a145dd2d7dfe4872b598c22da1816723, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/0fb97953b7f14c488e908dd92be95da1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a31bfa55ca6a4feba9bd051f74d3dc88, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/96fbeab0f9f24c428fcca75902567880, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1ed615d58c2741ac9c9f8861d704cd6b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/118da089e65f47709a9b883266587e38, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/efb6a3d215f5487eae98578cb249ba89, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c0a401d9f266447dad342d5da8d232e4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/451d9609efd542599de8cd458c4bee1a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/376b3061dfb14b6ca8a2005d27e25c7a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/2e764d6b7e634199854c385dbb4ca0ee, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4e6c240c29584f5bbf7c2d976ea95642] to archive 2024-11-11T23:54:25,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:54:25,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/6207fc4b3b0a4baaa597573de927436d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/6207fc4b3b0a4baaa597573de927436d 2024-11-11T23:54:25,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/dc415fdc23cd4c8caf08d6a865e03a4c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/dc415fdc23cd4c8caf08d6a865e03a4c 2024-11-11T23:54:25,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/90b920c6e64c469e920378db69a9721c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/90b920c6e64c469e920378db69a9721c 2024-11-11T23:54:25,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/43fb5f379335446e8132ebbbb75c4ddd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/43fb5f379335446e8132ebbbb75c4ddd 2024-11-11T23:54:25,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4ab6229a75114ef695be4a8adc0fc9be to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4ab6229a75114ef695be4a8adc0fc9be 2024-11-11T23:54:25,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/70bb24c8c53f41f383a258561c164476 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/70bb24c8c53f41f383a258561c164476 2024-11-11T23:54:25,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/ca54f03aeaf141fab0892a1dbf33ef0f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/ca54f03aeaf141fab0892a1dbf33ef0f 2024-11-11T23:54:25,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/572497a1c8bd48249e80cd8fec1bdcb4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/572497a1c8bd48249e80cd8fec1bdcb4 2024-11-11T23:54:25,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a145dd2d7dfe4872b598c22da1816723 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a145dd2d7dfe4872b598c22da1816723 2024-11-11T23:54:25,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/0fb97953b7f14c488e908dd92be95da1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/0fb97953b7f14c488e908dd92be95da1 2024-11-11T23:54:25,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a31bfa55ca6a4feba9bd051f74d3dc88 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/a31bfa55ca6a4feba9bd051f74d3dc88 2024-11-11T23:54:25,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/96fbeab0f9f24c428fcca75902567880 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/96fbeab0f9f24c428fcca75902567880 2024-11-11T23:54:25,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1ed615d58c2741ac9c9f8861d704cd6b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1ed615d58c2741ac9c9f8861d704cd6b 2024-11-11T23:54:25,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/118da089e65f47709a9b883266587e38 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/118da089e65f47709a9b883266587e38 2024-11-11T23:54:25,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/efb6a3d215f5487eae98578cb249ba89 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/efb6a3d215f5487eae98578cb249ba89 2024-11-11T23:54:25,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c0a401d9f266447dad342d5da8d232e4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c0a401d9f266447dad342d5da8d232e4 2024-11-11T23:54:25,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/451d9609efd542599de8cd458c4bee1a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/451d9609efd542599de8cd458c4bee1a 2024-11-11T23:54:25,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/376b3061dfb14b6ca8a2005d27e25c7a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/376b3061dfb14b6ca8a2005d27e25c7a 2024-11-11T23:54:25,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/2e764d6b7e634199854c385dbb4ca0ee to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/2e764d6b7e634199854c385dbb4ca0ee 2024-11-11T23:54:25,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4e6c240c29584f5bbf7c2d976ea95642 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/4e6c240c29584f5bbf7c2d976ea95642 2024-11-11T23:54:25,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/fb1c9e735c7e4e9c8e38905ce2f3d16a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/490c7d57f5464ab59938441f4387a08b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/f5ea4970c71a4365b5544b9abb07d80c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8142cdfd5b004ae6b6f4928529bd4bf3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/0c0fb5b971ed49ada5aefb6872cd3446, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/366d96c3068a478888424024341c27ac, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/dcc117440f8d4ceca53189e8ab39f327, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/344bdf59367542b7b9167f63aabdf2f8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/aab84cd12759469086bec5091a37748f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c02a47c6d4e84d588bdbd8e3b9269b22, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/a1f53ce4501648e5960911e38dc602b8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8f30ef0be25d4c1f96c5b3fb090e1c72, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c44922e9b20042818bf0136b9428d633, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/19c051ef42d74357895419acdecc8e20, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c4489fdb5c694fdb9f6e060e390cf78e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/324124cab9684f4495901ab9643c5a49, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/b5c6b056d6a843bfaf733e8099eafbda, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/5bd9600232934d9190c15346fe59b8ad, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/56ed9ce214d34fc6b2264c5590c2b6c7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/94b37ffa97394a3d92ba0af9f17fe4f4] to archive 2024-11-11T23:54:25,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:54:25,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/fb1c9e735c7e4e9c8e38905ce2f3d16a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/fb1c9e735c7e4e9c8e38905ce2f3d16a 2024-11-11T23:54:25,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/490c7d57f5464ab59938441f4387a08b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/490c7d57f5464ab59938441f4387a08b 2024-11-11T23:54:25,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/f5ea4970c71a4365b5544b9abb07d80c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/f5ea4970c71a4365b5544b9abb07d80c 2024-11-11T23:54:25,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8142cdfd5b004ae6b6f4928529bd4bf3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8142cdfd5b004ae6b6f4928529bd4bf3 2024-11-11T23:54:25,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/0c0fb5b971ed49ada5aefb6872cd3446 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/0c0fb5b971ed49ada5aefb6872cd3446 2024-11-11T23:54:25,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/366d96c3068a478888424024341c27ac to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/366d96c3068a478888424024341c27ac 2024-11-11T23:54:25,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/dcc117440f8d4ceca53189e8ab39f327 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/dcc117440f8d4ceca53189e8ab39f327 2024-11-11T23:54:25,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/344bdf59367542b7b9167f63aabdf2f8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/344bdf59367542b7b9167f63aabdf2f8 2024-11-11T23:54:25,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/aab84cd12759469086bec5091a37748f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/aab84cd12759469086bec5091a37748f 2024-11-11T23:54:25,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c02a47c6d4e84d588bdbd8e3b9269b22 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c02a47c6d4e84d588bdbd8e3b9269b22 2024-11-11T23:54:25,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/a1f53ce4501648e5960911e38dc602b8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/a1f53ce4501648e5960911e38dc602b8 2024-11-11T23:54:25,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8f30ef0be25d4c1f96c5b3fb090e1c72 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/8f30ef0be25d4c1f96c5b3fb090e1c72 2024-11-11T23:54:25,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c44922e9b20042818bf0136b9428d633 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c44922e9b20042818bf0136b9428d633 2024-11-11T23:54:25,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/19c051ef42d74357895419acdecc8e20 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/19c051ef42d74357895419acdecc8e20 2024-11-11T23:54:25,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c4489fdb5c694fdb9f6e060e390cf78e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/c4489fdb5c694fdb9f6e060e390cf78e 2024-11-11T23:54:25,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/324124cab9684f4495901ab9643c5a49 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/324124cab9684f4495901ab9643c5a49 2024-11-11T23:54:25,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/b5c6b056d6a843bfaf733e8099eafbda to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/b5c6b056d6a843bfaf733e8099eafbda 2024-11-11T23:54:25,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/5bd9600232934d9190c15346fe59b8ad to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/5bd9600232934d9190c15346fe59b8ad 2024-11-11T23:54:25,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/56ed9ce214d34fc6b2264c5590c2b6c7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/56ed9ce214d34fc6b2264c5590c2b6c7 2024-11-11T23:54:25,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/94b37ffa97394a3d92ba0af9f17fe4f4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/94b37ffa97394a3d92ba0af9f17fe4f4 2024-11-11T23:54:25,941 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=4 2024-11-11T23:54:25,941 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. 2024-11-11T23:54:25,941 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for d71f402ac6f3fd6fb0701033f3e67af7: Waiting for close lock at 1731369264191Running coprocessor pre-close hooks at 1731369264191Disabling compacts and flushes for region at 1731369264191Disabling writes for close at 1731369264191Obtaining lock to block concurrent updates at 1731369264191Preparing flush snapshotting stores in d71f402ac6f3fd6fb0701033f3e67af7 at 1731369264191Finished memstore snapshotting TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731369264192 (+1 ms)Flushing stores of TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7. at 1731369264193 (+1 ms)Flushing d71f402ac6f3fd6fb0701033f3e67af7/A: creating writer at 1731369264193Flushing mob file d71f402ac6f3fd6fb0701033f3e67af7/A: appending metadata at 1731369264198 (+5 ms)Flushing mob file d71f402ac6f3fd6fb0701033f3e67af7/A: closing flushed file at 1731369264198Flushing d71f402ac6f3fd6fb0701033f3e67af7/A: appending metadata at 1731369264609 (+411 ms)Flushing d71f402ac6f3fd6fb0701033f3e67af7/A: closing flushed file at 1731369264609Flushing d71f402ac6f3fd6fb0701033f3e67af7/B: creating writer at 1731369265024 (+415 ms)Flushing d71f402ac6f3fd6fb0701033f3e67af7/B: appending metadata at 1731369265027 (+3 ms)Flushing d71f402ac6f3fd6fb0701033f3e67af7/B: closing flushed file at 1731369265027Flushing d71f402ac6f3fd6fb0701033f3e67af7/C: creating writer at 1731369265438 (+411 ms)Flushing d71f402ac6f3fd6fb0701033f3e67af7/C: appending metadata at 1731369265442 (+4 ms)Flushing d71f402ac6f3fd6fb0701033f3e67af7/C: closing flushed file at 1731369265442Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e451170: reopening flushed file at 1731369265855 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2802cb38: reopening flushed file at 1731369265864 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@342a4ee6: reopening flushed file at 1731369265870 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d71f402ac6f3fd6fb0701033f3e67af7 in 1685ms, sequenceid=433, compaction requested=true at 1731369265876 (+6 ms)Writing region close event to WAL at 1731369265938 (+62 ms)Running coprocessor post-close hooks at 1731369265941 (+3 ms)Closed at 1731369265941 2024-11-11T23:54:25,943 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:25,944 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=d71f402ac6f3fd6fb0701033f3e67af7, regionState=CLOSED 2024-11-11T23:54:25,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:54:25,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T23:54:25,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure d71f402ac6f3fd6fb0701033f3e67af7, server=b70d29849faf,42393,1731369201430 in 1.9070 sec 2024-11-11T23:54:25,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-11T23:54:25,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d71f402ac6f3fd6fb0701033f3e67af7, UNASSIGN in 1.9120 sec 2024-11-11T23:54:25,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T23:54:25,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-11T23:54:25,954 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369265954"}]},"ts":"1731369265954"} 2024-11-11T23:54:25,955 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T23:54:25,955 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T23:54:25,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9270 sec 2024-11-11T23:54:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T23:54:26,176 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:26,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T23:54:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:26,180 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T23:54:26,182 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:26,187 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,193 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/recovered.edits] 2024-11-11T23:54:26,195 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/13ef7669b8024ef9a59555849a225c08 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/13ef7669b8024ef9a59555849a225c08 2024-11-11T23:54:26,196 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/4a1b1224721b4fa7b35cc584e19a1e67 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/4a1b1224721b4fa7b35cc584e19a1e67 2024-11-11T23:54:26,197 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/b1cdc3b9acd44b9cb37d726433e620fa to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/A/b1cdc3b9acd44b9cb37d726433e620fa 2024-11-11T23:54:26,199 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1de27900d33843d583e22b8463b64e4e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/1de27900d33843d583e22b8463b64e4e 2024-11-11T23:54:26,200 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/671b1ecc8fba452a99f37c9032933c23 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/671b1ecc8fba452a99f37c9032933c23 2024-11-11T23:54:26,201 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c109ae319fe9401ab4148aa01d306f69 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/B/c109ae319fe9401ab4148aa01d306f69 2024-11-11T23:54:26,203 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/3f49e01b69bb42ed89b1c7eaaef187f5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/3f49e01b69bb42ed89b1c7eaaef187f5 2024-11-11T23:54:26,204 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/4a8e21f0269f457eb1aceac9b096d3ce to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/4a8e21f0269f457eb1aceac9b096d3ce 2024-11-11T23:54:26,205 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/e8ccb3464b474777a2a24ba3da7dfdf8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/C/e8ccb3464b474777a2a24ba3da7dfdf8 2024-11-11T23:54:26,207 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/recovered.edits/436.seqid to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7/recovered.edits/436.seqid 2024-11-11T23:54:26,207 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,207 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T23:54:26,208 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T23:54:26,208 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T23:54:26,212 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111103da499855504d918d1cc42e18309c4c_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111103da499855504d918d1cc42e18309c4c_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,213 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110fde8d2d3746406e8b8e6a192584fbfc_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110fde8d2d3746406e8b8e6a192584fbfc_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,214 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111115d3bf5b8d744859874b53eb231cd9ec_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111115d3bf5b8d744859874b53eb231cd9ec_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,215 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b7dbd3646354a1fa202bbea3aa938f3_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b7dbd3646354a1fa202bbea3aa938f3_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,216 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111127b6f72509ff46f2a14f1383d8eba6b1_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111127b6f72509ff46f2a14f1383d8eba6b1_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,217 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111135af3696169c4c34a646f0cbdba9251d_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111135af3696169c4c34a646f0cbdba9251d_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,218 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113d74e4d1b8f34b1c8f6c498c282e5e18_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113d74e4d1b8f34b1c8f6c498c282e5e18_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,219 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155b347e989834cefb13ccc51721014fc_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155b347e989834cefb13ccc51721014fc_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,220 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166f1ad31af9c419182bc1f8c2b9e7d81_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166f1ad31af9c419182bc1f8c2b9e7d81_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,221 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ae8037726084380a396ecb247a3d696_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ae8037726084380a396ecb247a3d696_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,222 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111726256e007cf4ffbb9ee644603348a53_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111726256e007cf4ffbb9ee644603348a53_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,223 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111184dc67d1d8a24884a64a0eb4a76001a4_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111184dc67d1d8a24884a64a0eb4a76001a4_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,223 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0b14532abb24972a104fb4a938cd2ff_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0b14532abb24972a104fb4a938cd2ff_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,224 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d1b4b1bc4dee4dd09e29f866843b31b3_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d1b4b1bc4dee4dd09e29f866843b31b3_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,225 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d1db13789c6d4129b80c3d80d6f28599_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d1db13789c6d4129b80c3d80d6f28599_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,226 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d939a5138eed4340bf4698a3e5b22092_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d939a5138eed4340bf4698a3e5b22092_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,227 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f89c52d5fb794d51ae7a48bb61dde00c_d71f402ac6f3fd6fb0701033f3e67af7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f89c52d5fb794d51ae7a48bb61dde00c_d71f402ac6f3fd6fb0701033f3e67af7 2024-11-11T23:54:26,227 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T23:54:26,229 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:26,231 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T23:54:26,233 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T23:54:26,233 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:26,233 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T23:54:26,234 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731369266234"}]},"ts":"9223372036854775807"} 2024-11-11T23:54:26,235 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T23:54:26,235 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d71f402ac6f3fd6fb0701033f3e67af7, NAME => 'TestAcidGuarantees,,1731369233393.d71f402ac6f3fd6fb0701033f3e67af7.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T23:54:26,235 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T23:54:26,236 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731369266235"}]},"ts":"9223372036854775807"} 2024-11-11T23:54:26,238 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T23:54:26,239 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:26,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-11T23:54:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T23:54:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T23:54:26,285 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:26,296 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=530 (was 547), ProcessCount=11 (was 11), AvailableMemoryMB=4209 (was 4816) 2024-11-11T23:54:26,307 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=530, ProcessCount=11, AvailableMemoryMB=4209 2024-11-11T23:54:26,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:54:26,309 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:54:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:26,312 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:54:26,312 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:26,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-11T23:54:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-11T23:54:26,313 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:54:26,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742043_1219 (size=1061) 2024-11-11T23:54:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-11T23:54:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-11T23:54:26,727 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 2024-11-11T23:54:26,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742044_1220 (size=53) 2024-11-11T23:54:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-11T23:54:27,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:54:27,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1311b3c706ba19435848d682254b17d6, disabling compactions & flushes 2024-11-11T23:54:27,138 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:27,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:27,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. after waiting 0 ms 2024-11-11T23:54:27,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:27,138 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:27,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1311b3c706ba19435848d682254b17d6: Waiting for close lock at 1731369267138Disabling compacts and flushes for region at 1731369267138Disabling writes for close at 1731369267138Writing region close event to WAL at 1731369267138Closed at 1731369267138 2024-11-11T23:54:27,140 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:54:27,140 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731369267140"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369267140"}]},"ts":"1731369267140"} 2024-11-11T23:54:27,142 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:54:27,143 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:54:27,143 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369267143"}]},"ts":"1731369267143"} 2024-11-11T23:54:27,145 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T23:54:27,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, ASSIGN}] 2024-11-11T23:54:27,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, ASSIGN 2024-11-11T23:54:27,147 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, ASSIGN; state=OFFLINE, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=false 2024-11-11T23:54:27,299 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=1311b3c706ba19435848d682254b17d6, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:27,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, ASSIGN because future has completed 2024-11-11T23:54:27,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:54:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-11T23:54:27,464 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:27,465 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:54:27,465 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,465 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:54:27,465 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,466 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,468 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,469 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:27,470 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1311b3c706ba19435848d682254b17d6 columnFamilyName A 2024-11-11T23:54:27,470 DEBUG [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:27,471 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.HStore(327): Store=1311b3c706ba19435848d682254b17d6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:27,471 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,472 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:27,472 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1311b3c706ba19435848d682254b17d6 columnFamilyName B 2024-11-11T23:54:27,472 DEBUG [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:27,473 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.HStore(327): Store=1311b3c706ba19435848d682254b17d6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:27,473 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,474 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:27,474 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1311b3c706ba19435848d682254b17d6 columnFamilyName C 2024-11-11T23:54:27,474 DEBUG [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:27,475 INFO [StoreOpener-1311b3c706ba19435848d682254b17d6-1 {}] regionserver.HStore(327): Store=1311b3c706ba19435848d682254b17d6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:27,475 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:27,475 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,476 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,476 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,477 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,477 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,477 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:54:27,478 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,480 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:54:27,481 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 1311b3c706ba19435848d682254b17d6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60299631, jitterRate=-0.10146547853946686}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:54:27,481 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:27,482 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 1311b3c706ba19435848d682254b17d6: Running coprocessor pre-open hook at 1731369267466Writing region info on filesystem at 1731369267466Initializing all the Stores at 1731369267467 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369267467Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369267467Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369267467Cleaning up temporary data from old regions at 1731369267477 (+10 ms)Running coprocessor post-open hooks at 1731369267481 (+4 ms)Region opened successfully at 1731369267482 (+1 ms) 2024-11-11T23:54:27,482 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., pid=57, masterSystemTime=1731369267459 2024-11-11T23:54:27,484 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:27,484 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:27,485 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=1311b3c706ba19435848d682254b17d6, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:27,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:54:27,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T23:54:27,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 in 183 msec 2024-11-11T23:54:27,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-11T23:54:27,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, ASSIGN in 343 msec 2024-11-11T23:54:27,491 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:54:27,491 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369267491"}]},"ts":"1731369267491"} 2024-11-11T23:54:27,492 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T23:54:27,493 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:54:27,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-11T23:54:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-11T23:54:28,465 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:28,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,468 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637888f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4215e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,470 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,473 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T23:54:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b70d29849faf,37327,1731369200577 2024-11-11T23:54:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61a1e1c4 2024-11-11T23:54:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T23:54:28,474 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T23:54:28,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,476 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483e2e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,478 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d971a56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,480 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b009d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,481 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656cbb3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,483 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f0c195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,485 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ecfd2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,486 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@474f0b9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,487 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fef0138, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,490 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28cb693a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,491 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ba62b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,493 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7444fc2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4c12e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b449e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,497 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db28e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,498 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481add00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,500 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51efc443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,501 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b89271d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,502 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de2bcbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:28,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:28,504 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431c0b45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:28,505 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca1be6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:28,506 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,506 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,506 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,506 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,506 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,507 DEBUG [Thread-1047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,507 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,507 DEBUG [Thread-1039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,507 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,507 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,508 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-11T23:54:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T23:54:28,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:28,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:28,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,511 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:28,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,513 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:28,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:54:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:28,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:54:28,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:28,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:28,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:28,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:28,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:28,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:28,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/e6f8a88727f94855b371dd2771509ed1 is 50, key is test_row_0/A:col10/1731369268520/Put/seqid=0 2024-11-11T23:54:28,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369268537, max timestamp=1731369268557Num uniques 859; 2024-11-11T23:54:28,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:54:28,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369268537, max timestamp=1731369268557Num uniques 859; 2024-11-11T23:54:28,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:54:28,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369268549, max timestamp=1731369268557Num uniques 859; 2024-11-11T23:54:28,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:54:28,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742045_1221 (size=12001) 2024-11-11T23:54:28,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/e6f8a88727f94855b371dd2771509ed1 2024-11-11T23:54:28,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369328583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369328581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369328585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369328586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369328587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/b7ede7d41f5c49dda0ffba66b7ea802d is 50, key is test_row_0/B:col10/1731369268520/Put/seqid=0 2024-11-11T23:54:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T23:54:28,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742046_1222 (size=12001) 2024-11-11T23:54:28,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/b7ede7d41f5c49dda0ffba66b7ea802d 2024-11-11T23:54:28,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a610ae9e0967482487174fb77056bb29 is 50, key is test_row_0/C:col10/1731369268520/Put/seqid=0 2024-11-11T23:54:28,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T23:54:28,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:28,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:28,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:28,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:28,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:28,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742047_1223 (size=12001) 2024-11-11T23:54:28,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369328694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369328694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369328704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369328704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369328705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T23:54:28,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:28,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:28,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:28,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:28,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T23:54:28,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369328904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369328904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369328915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369328915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369328915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:28,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T23:54:28,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:28,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:28,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:28,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:28,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:29,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a610ae9e0967482487174fb77056bb29 2024-11-11T23:54:29,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/e6f8a88727f94855b371dd2771509ed1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e6f8a88727f94855b371dd2771509ed1 2024-11-11T23:54:29,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e6f8a88727f94855b371dd2771509ed1, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T23:54:29,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/b7ede7d41f5c49dda0ffba66b7ea802d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/b7ede7d41f5c49dda0ffba66b7ea802d 2024-11-11T23:54:29,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/b7ede7d41f5c49dda0ffba66b7ea802d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T23:54:29,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a610ae9e0967482487174fb77056bb29 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a610ae9e0967482487174fb77056bb29 2024-11-11T23:54:29,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a610ae9e0967482487174fb77056bb29, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T23:54:29,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 1311b3c706ba19435848d682254b17d6 in 592ms, sequenceid=12, compaction requested=false 2024-11-11T23:54:29,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:29,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T23:54:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,129 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=131.94 KB heapSize=370.53 KB 2024-11-11T23:54:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:29,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:29,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T23:54:29,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/a8505b490e95498da748f5121064d386 is 50, key is test_row_0/A:col10/1731369268586/Put/seqid=0 2024-11-11T23:54:29,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742048_1224 (size=12001) 2024-11-11T23:54:29,164 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/a8505b490e95498da748f5121064d386 2024-11-11T23:54:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/fc61fcfb853c46fb9e88797600650fe7 is 50, key is test_row_0/B:col10/1731369268586/Put/seqid=0 2024-11-11T23:54:29,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742049_1225 (size=12001) 2024-11-11T23:54:29,192 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/fc61fcfb853c46fb9e88797600650fe7 2024-11-11T23:54:29,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a15f73fc8ba84d0d9fe496ba569b90da is 50, key is test_row_0/C:col10/1731369268586/Put/seqid=0 2024-11-11T23:54:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:29,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:29,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742050_1226 (size=12001) 2024-11-11T23:54:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369329226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,237 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a15f73fc8ba84d0d9fe496ba569b90da 2024-11-11T23:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369329234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369329236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369329236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369329237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/a8505b490e95498da748f5121064d386 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/a8505b490e95498da748f5121064d386 2024-11-11T23:54:29,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/a8505b490e95498da748f5121064d386, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T23:54:29,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/fc61fcfb853c46fb9e88797600650fe7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/fc61fcfb853c46fb9e88797600650fe7 2024-11-11T23:54:29,260 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/fc61fcfb853c46fb9e88797600650fe7, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T23:54:29,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a15f73fc8ba84d0d9fe496ba569b90da as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a15f73fc8ba84d0d9fe496ba569b90da 2024-11-11T23:54:29,269 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a15f73fc8ba84d0d9fe496ba569b90da, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T23:54:29,269 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~370.48 KB/379376, currentSize=53.67 KB/54960 for 1311b3c706ba19435848d682254b17d6 in 140ms, sequenceid=47, compaction requested=false 2024-11-11T23:54:29,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:29,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-11T23:54:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-11T23:54:29,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T23:54:29,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 758 msec 2024-11-11T23:54:29,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 769 msec 2024-11-11T23:54:29,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T23:54:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:29,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:29,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:29,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/4af9c2d5bd004442a74e4db3ac1f1e05 is 50, key is test_row_0/A:col10/1731369269234/Put/seqid=0 2024-11-11T23:54:29,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742051_1227 (size=14341) 2024-11-11T23:54:29,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/4af9c2d5bd004442a74e4db3ac1f1e05 2024-11-11T23:54:29,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369329374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369329375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369329377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369329379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369329383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/4583c03d70124bb2b1432e5e2323c5f7 is 50, key is test_row_0/B:col10/1731369269234/Put/seqid=0 2024-11-11T23:54:29,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742052_1228 (size=12001) 2024-11-11T23:54:29,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/4583c03d70124bb2b1432e5e2323c5f7 2024-11-11T23:54:29,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/25534b251c35404d82d80477926c1f20 is 50, key is test_row_0/C:col10/1731369269234/Put/seqid=0 2024-11-11T23:54:29,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742053_1229 (size=12001) 2024-11-11T23:54:29,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/25534b251c35404d82d80477926c1f20 2024-11-11T23:54:29,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/4af9c2d5bd004442a74e4db3ac1f1e05 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/4af9c2d5bd004442a74e4db3ac1f1e05 2024-11-11T23:54:29,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/4af9c2d5bd004442a74e4db3ac1f1e05, entries=200, sequenceid=59, filesize=14.0 K 2024-11-11T23:54:29,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/4583c03d70124bb2b1432e5e2323c5f7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4583c03d70124bb2b1432e5e2323c5f7 2024-11-11T23:54:29,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4583c03d70124bb2b1432e5e2323c5f7, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T23:54:29,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/25534b251c35404d82d80477926c1f20 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/25534b251c35404d82d80477926c1f20 2024-11-11T23:54:29,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/25534b251c35404d82d80477926c1f20, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T23:54:29,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1311b3c706ba19435848d682254b17d6 in 137ms, sequenceid=59, compaction requested=true 2024-11-11T23:54:29,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:29,485 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:29,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:29,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:29,485 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:29,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:29,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:29,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:29,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:29,488 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:29,488 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:29,488 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:29,488 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:29,488 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,488 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,488 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e6f8a88727f94855b371dd2771509ed1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/a8505b490e95498da748f5121064d386, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/4af9c2d5bd004442a74e4db3ac1f1e05] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=37.4 K 2024-11-11T23:54:29,488 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/b7ede7d41f5c49dda0ffba66b7ea802d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/fc61fcfb853c46fb9e88797600650fe7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4583c03d70124bb2b1432e5e2323c5f7] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=35.2 K 2024-11-11T23:54:29,489 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f8a88727f94855b371dd2771509ed1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369268520 2024-11-11T23:54:29,489 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ede7d41f5c49dda0ffba66b7ea802d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369268520 2024-11-11T23:54:29,489 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8505b490e95498da748f5121064d386, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731369268581 2024-11-11T23:54:29,490 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting fc61fcfb853c46fb9e88797600650fe7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731369268581 2024-11-11T23:54:29,490 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af9c2d5bd004442a74e4db3ac1f1e05, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731369269230 2024-11-11T23:54:29,490 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4583c03d70124bb2b1432e5e2323c5f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731369269230 2024-11-11T23:54:29,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369269359, max timestamp=1731369269496Num uniques 859; 2024-11-11T23:54:29,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:29,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-11-11T23:54:29,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731369269359, max timestamp=1731369269496Num uniques 859; 2024-11-11T23:54:29,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T23:54:29,505 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:29,506 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/3101c125ad0a403f9ce519acea1d011c is 50, key is test_row_0/B:col10/1731369269234/Put/seqid=0 2024-11-11T23:54:29,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731369269359, max timestamp=1731369269498Num uniques 859; 2024-11-11T23:54:29,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T23:54:29,506 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#189 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:29,507 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/21da2d9baf1644e5afc29a8db03e4a8f is 50, key is test_row_0/A:col10/1731369269234/Put/seqid=0 2024-11-11T23:54:29,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:29,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:29,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:29,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:29,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:29,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:29,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/dda69eb0e8f147eeb04688e5d97fabc4 is 50, key is test_row_0/A:col10/1731369269501/Put/seqid=0 2024-11-11T23:54:29,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731369269520, max timestamp=1731369269542Num uniques 859; 2024-11-11T23:54:29,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T23:54:29,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731369269520, max timestamp=1731369269542Num uniques 859; 2024-11-11T23:54:29,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731369269520, max timestamp=1731369269542Num uniques 859; 2024-11-11T23:54:29,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T23:54:29,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T23:54:29,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742055_1231 (size=12104) 2024-11-11T23:54:29,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742056_1232 (size=12001) 2024-11-11T23:54:29,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742054_1230 (size=12104) 2024-11-11T23:54:29,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369329566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369329567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369329575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369329576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369329577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T23:54:29,645 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:29,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-11T23:54:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T23:54:29,647 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:29,647 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:29,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:29,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369329685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369329685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369329695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369329695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369329695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T23:54:29,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T23:54:29,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:29,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:29,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:29,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369329894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369329894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369329904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369329904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369329904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:29,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T23:54:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:29,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/dda69eb0e8f147eeb04688e5d97fabc4 2024-11-11T23:54:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T23:54:29,965 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/21da2d9baf1644e5afc29a8db03e4a8f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/21da2d9baf1644e5afc29a8db03e4a8f 2024-11-11T23:54:29,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/161f2d23e77b43f181dd85eeb99d14ba is 50, key is test_row_0/B:col10/1731369269501/Put/seqid=0 2024-11-11T23:54:29,977 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into 21da2d9baf1644e5afc29a8db03e4a8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:29,977 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:29,977 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=13, startTime=1731369269484; duration=0sec 2024-11-11T23:54:29,977 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:29,977 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:29,977 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:29,978 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:29,978 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:29,978 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:29,979 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a610ae9e0967482487174fb77056bb29, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a15f73fc8ba84d0d9fe496ba569b90da, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/25534b251c35404d82d80477926c1f20] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=35.2 K 2024-11-11T23:54:29,979 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a610ae9e0967482487174fb77056bb29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369268520 2024-11-11T23:54:29,979 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a15f73fc8ba84d0d9fe496ba569b90da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731369268581 2024-11-11T23:54:29,980 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25534b251c35404d82d80477926c1f20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731369269230 2024-11-11T23:54:29,988 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/3101c125ad0a403f9ce519acea1d011c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/3101c125ad0a403f9ce519acea1d011c 2024-11-11T23:54:29,994 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into 3101c125ad0a403f9ce519acea1d011c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:29,994 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:29,994 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=13, startTime=1731369269485; duration=0sec 2024-11-11T23:54:29,995 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:29,995 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:29,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742057_1233 (size=12001) 2024-11-11T23:54:29,998 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:29,999 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/9fdfd2a4d08e4155861b600c24ef566f is 50, key is test_row_0/C:col10/1731369269234/Put/seqid=0 2024-11-11T23:54:30,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742058_1234 (size=12104) 2024-11-11T23:54:30,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T23:54:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369330205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369330205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369330214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369330214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369330214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T23:54:30,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:30,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T23:54:30,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.50 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/161f2d23e77b43f181dd85eeb99d14ba 2024-11-11T23:54:30,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b143b4dce8194423b3646ad485478b45 is 50, key is test_row_0/C:col10/1731369269501/Put/seqid=0 2024-11-11T23:54:30,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742059_1235 (size=12001) 2024-11-11T23:54:30,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T23:54:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,415 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/9fdfd2a4d08e4155861b600c24ef566f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/9fdfd2a4d08e4155861b600c24ef566f 2024-11-11T23:54:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,420 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 9fdfd2a4d08e4155861b600c24ef566f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:30,420 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:30,420 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=13, startTime=1731369269485; duration=0sec 2024-11-11T23:54:30,420 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:30,420 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:30,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T23:54:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369330714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369330714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T23:54:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:30,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369330724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369330725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369330725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T23:54:30,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.50 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b143b4dce8194423b3646ad485478b45 2024-11-11T23:54:30,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/dda69eb0e8f147eeb04688e5d97fabc4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/dda69eb0e8f147eeb04688e5d97fabc4 2024-11-11T23:54:30,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/dda69eb0e8f147eeb04688e5d97fabc4, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T23:54:30,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/161f2d23e77b43f181dd85eeb99d14ba as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/161f2d23e77b43f181dd85eeb99d14ba 2024-11-11T23:54:30,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/161f2d23e77b43f181dd85eeb99d14ba, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T23:54:30,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b143b4dce8194423b3646ad485478b45 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b143b4dce8194423b3646ad485478b45 2024-11-11T23:54:30,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b143b4dce8194423b3646ad485478b45, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T23:54:30,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~52.88 KB/54150, heapSize ~144.05 KB/147504, currentSize=140.89 KB/144270 for 1311b3c706ba19435848d682254b17d6 in 1340ms, sequenceid=84, compaction requested=false 2024-11-11T23:54:30,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:30,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T23:54:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:30,873 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-11T23:54:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/686a8482b74e488aa4e90dc375e96536 is 50, key is test_row_0/A:col10/1731369269562/Put/seqid=0 2024-11-11T23:54:30,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742060_1236 (size=12001) 2024-11-11T23:54:31,282 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/686a8482b74e488aa4e90dc375e96536 2024-11-11T23:54:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/c0cd556df4c2499b82471d6399682986 is 50, key is test_row_0/B:col10/1731369269562/Put/seqid=0 2024-11-11T23:54:31,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742061_1237 (size=12001) 2024-11-11T23:54:31,465 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:54:31,694 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/c0cd556df4c2499b82471d6399682986 2024-11-11T23:54:31,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/91a7cb4a02404ca0aeb057f458fe8679 is 50, key is test_row_0/C:col10/1731369269562/Put/seqid=0 2024-11-11T23:54:31,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742062_1238 (size=12001) 2024-11-11T23:54:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:31,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369331739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369331740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369331740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:31,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369331743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369331749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,750 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 1311b3c706ba19435848d682254b17d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T23:54:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369331855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369331855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369331855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369331855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369332065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369332065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369332065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369332065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,106 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/91a7cb4a02404ca0aeb057f458fe8679 2024-11-11T23:54:32,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/686a8482b74e488aa4e90dc375e96536 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/686a8482b74e488aa4e90dc375e96536 2024-11-11T23:54:32,119 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/686a8482b74e488aa4e90dc375e96536, entries=150, sequenceid=121, filesize=11.7 K 2024-11-11T23:54:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/c0cd556df4c2499b82471d6399682986 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c0cd556df4c2499b82471d6399682986 2024-11-11T23:54:32,123 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c0cd556df4c2499b82471d6399682986, entries=150, sequenceid=121, filesize=11.7 K 2024-11-11T23:54:32,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/91a7cb4a02404ca0aeb057f458fe8679 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/91a7cb4a02404ca0aeb057f458fe8679 2024-11-11T23:54:32,130 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/91a7cb4a02404ca0aeb057f458fe8679, entries=150, sequenceid=121, filesize=11.7 K 2024-11-11T23:54:32,131 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 1311b3c706ba19435848d682254b17d6 in 1257ms, sequenceid=121, compaction requested=true 2024-11-11T23:54:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-11T23:54:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-11T23:54:32,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T23:54:32,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4860 sec 2024-11-11T23:54:32,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.4900 sec 2024-11-11T23:54:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:32,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T23:54:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:32,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/adaefb2eab104bad9a798bbf20df4830 is 50, key is test_row_0/A:col10/1731369272377/Put/seqid=0 2024-11-11T23:54:32,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742063_1239 (size=12151) 2024-11-11T23:54:32,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369332407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369332408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369332413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369332413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369332524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369332524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369332524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369332524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369332735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369332735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369332735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369332735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:32,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/adaefb2eab104bad9a798bbf20df4830 2024-11-11T23:54:32,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/2ecdcfd3c8864c98b33b3c1dabed23ca is 50, key is test_row_0/B:col10/1731369272377/Put/seqid=0 2024-11-11T23:54:32,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742064_1240 (size=12151) 2024-11-11T23:54:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/2ecdcfd3c8864c98b33b3c1dabed23ca 2024-11-11T23:54:33,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/991561d21c5c4db6b794ddf58b70a465 is 50, key is test_row_0/C:col10/1731369272377/Put/seqid=0 2024-11-11T23:54:33,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742065_1241 (size=12151) 2024-11-11T23:54:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369333555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369333555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369333555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369333555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/991561d21c5c4db6b794ddf58b70a465 2024-11-11T23:54:33,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/adaefb2eab104bad9a798bbf20df4830 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/adaefb2eab104bad9a798bbf20df4830 2024-11-11T23:54:33,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/adaefb2eab104bad9a798bbf20df4830, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T23:54:33,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/2ecdcfd3c8864c98b33b3c1dabed23ca as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2ecdcfd3c8864c98b33b3c1dabed23ca 2024-11-11T23:54:33,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2ecdcfd3c8864c98b33b3c1dabed23ca, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T23:54:33,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/991561d21c5c4db6b794ddf58b70a465 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/991561d21c5c4db6b794ddf58b70a465 2024-11-11T23:54:33,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/991561d21c5c4db6b794ddf58b70a465, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T23:54:33,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1311b3c706ba19435848d682254b17d6 in 1269ms, sequenceid=133, compaction requested=true 2024-11-11T23:54:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:33,647 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:33,647 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:33,648 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:33,648 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:33,648 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:33,649 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:33,649 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/21da2d9baf1644e5afc29a8db03e4a8f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/dda69eb0e8f147eeb04688e5d97fabc4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/686a8482b74e488aa4e90dc375e96536, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/adaefb2eab104bad9a798bbf20df4830] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=47.1 K 2024-11-11T23:54:33,649 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:33,649 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:33,649 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/3101c125ad0a403f9ce519acea1d011c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/161f2d23e77b43f181dd85eeb99d14ba, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c0cd556df4c2499b82471d6399682986, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2ecdcfd3c8864c98b33b3c1dabed23ca] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=47.1 K 2024-11-11T23:54:33,649 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3101c125ad0a403f9ce519acea1d011c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731369269230 2024-11-11T23:54:33,649 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 21da2d9baf1644e5afc29a8db03e4a8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731369269230 2024-11-11T23:54:33,650 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting dda69eb0e8f147eeb04688e5d97fabc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731369269500 2024-11-11T23:54:33,650 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 161f2d23e77b43f181dd85eeb99d14ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731369269500 2024-11-11T23:54:33,650 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0cd556df4c2499b82471d6399682986, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731369269562 2024-11-11T23:54:33,650 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 686a8482b74e488aa4e90dc375e96536, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731369269562 2024-11-11T23:54:33,651 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ecdcfd3c8864c98b33b3c1dabed23ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731369271739 2024-11-11T23:54:33,651 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting adaefb2eab104bad9a798bbf20df4830, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731369271739 2024-11-11T23:54:33,661 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:33,661 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:33,662 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/52b005d9ff7e42628724a54b01b9c933 is 50, key is test_row_0/A:col10/1731369272377/Put/seqid=0 2024-11-11T23:54:33,662 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/2d9f900350d1413193f0b0c2fe6598d5 is 50, key is test_row_0/B:col10/1731369272377/Put/seqid=0 2024-11-11T23:54:33,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742066_1242 (size=12391) 2024-11-11T23:54:33,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742067_1243 (size=12391) 2024-11-11T23:54:33,688 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/2d9f900350d1413193f0b0c2fe6598d5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2d9f900350d1413193f0b0c2fe6598d5 2024-11-11T23:54:33,695 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into 2d9f900350d1413193f0b0c2fe6598d5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:33,695 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:33,695 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=12, startTime=1731369273647; duration=0sec 2024-11-11T23:54:33,695 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:33,695 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:33,695 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:33,697 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:33,697 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:33,697 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:33,697 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/9fdfd2a4d08e4155861b600c24ef566f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b143b4dce8194423b3646ad485478b45, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/91a7cb4a02404ca0aeb057f458fe8679, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/991561d21c5c4db6b794ddf58b70a465] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=47.1 K 2024-11-11T23:54:33,698 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fdfd2a4d08e4155861b600c24ef566f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731369269230 2024-11-11T23:54:33,699 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b143b4dce8194423b3646ad485478b45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731369269500 2024-11-11T23:54:33,699 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a7cb4a02404ca0aeb057f458fe8679, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731369269562 2024-11-11T23:54:33,700 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991561d21c5c4db6b794ddf58b70a465, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731369271739 2024-11-11T23:54:33,714 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#203 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:33,714 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/506396762cc44f90a215db2af73f9578 is 50, key is test_row_0/C:col10/1731369272377/Put/seqid=0 2024-11-11T23:54:33,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742068_1244 (size=12391) 2024-11-11T23:54:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:33,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T23:54:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:33,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/994c070fd5c1428880d753aa9f14317f is 50, key is test_row_0/A:col10/1731369272412/Put/seqid=0 2024-11-11T23:54:33,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742069_1245 (size=14541) 2024-11-11T23:54:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T23:54:33,804 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:33,805 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-11T23:54:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T23:54:33,806 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:33,807 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:33,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:33,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369333821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T23:54:33,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369333934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:33,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T23:54:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:33,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,080 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/52b005d9ff7e42628724a54b01b9c933 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/52b005d9ff7e42628724a54b01b9c933 2024-11-11T23:54:34,084 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into 52b005d9ff7e42628724a54b01b9c933(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:34,084 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:34,084 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=12, startTime=1731369273647; duration=0sec 2024-11-11T23:54:34,084 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:34,084 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:34,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T23:54:34,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:34,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T23:54:34,136 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/506396762cc44f90a215db2af73f9578 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/506396762cc44f90a215db2af73f9578 2024-11-11T23:54:34,141 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 506396762cc44f90a215db2af73f9578(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:34,141 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:34,141 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=12, startTime=1731369273647; duration=0sec 2024-11-11T23:54:34,142 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:34,142 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:34,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369334144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:34,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/994c070fd5c1428880d753aa9f14317f 2024-11-11T23:54:34,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/11c7102677b34a98920e099b7fdfee12 is 50, key is test_row_0/B:col10/1731369272412/Put/seqid=0 2024-11-11T23:54:34,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742070_1246 (size=12151) 2024-11-11T23:54:34,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T23:54:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T23:54:34,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:34,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T23:54:34,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369334454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:34,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369334564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369334564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:34,568 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 1311b3c706ba19435848d682254b17d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:34,568 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 1311b3c706ba19435848d682254b17d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:34,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T23:54:34,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:34,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369334575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369334575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:34,579 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 1311b3c706ba19435848d682254b17d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:34,579 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 1311b3c706ba19435848d682254b17d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:34,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/11c7102677b34a98920e099b7fdfee12 2024-11-11T23:54:34,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c93d53d214d445a4bba038bd4b8c852a is 50, key is test_row_0/C:col10/1731369272412/Put/seqid=0 2024-11-11T23:54:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742071_1247 (size=12151) 2024-11-11T23:54:34,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c93d53d214d445a4bba038bd4b8c852a 2024-11-11T23:54:34,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/994c070fd5c1428880d753aa9f14317f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/994c070fd5c1428880d753aa9f14317f 2024-11-11T23:54:34,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/994c070fd5c1428880d753aa9f14317f, entries=200, sequenceid=155, filesize=14.2 K 2024-11-11T23:54:34,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/11c7102677b34a98920e099b7fdfee12 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/11c7102677b34a98920e099b7fdfee12 2024-11-11T23:54:34,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/11c7102677b34a98920e099b7fdfee12, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T23:54:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c93d53d214d445a4bba038bd4b8c852a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c93d53d214d445a4bba038bd4b8c852a 2024-11-11T23:54:34,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c93d53d214d445a4bba038bd4b8c852a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T23:54:34,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 1311b3c706ba19435848d682254b17d6 in 870ms, sequenceid=155, compaction requested=false 2024-11-11T23:54:34,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:34,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T23:54:34,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,722 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T23:54:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:34,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/fc385286f7474265a65d6a59a8285888 is 50, key is test_row_0/A:col10/1731369273790/Put/seqid=0 2024-11-11T23:54:34,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742072_1248 (size=12151) 2024-11-11T23:54:34,755 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/fc385286f7474265a65d6a59a8285888 2024-11-11T23:54:34,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/2f09dd50c7ba4d688833ced7ba183001 is 50, key is test_row_0/B:col10/1731369273790/Put/seqid=0 2024-11-11T23:54:34,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742073_1249 (size=12151) 2024-11-11T23:54:34,774 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/2f09dd50c7ba4d688833ced7ba183001 2024-11-11T23:54:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/15802501efb4423486e996c6260b9c43 is 50, key is test_row_0/C:col10/1731369273790/Put/seqid=0 2024-11-11T23:54:34,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742074_1250 (size=12151) 2024-11-11T23:54:34,788 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/15802501efb4423486e996c6260b9c43 2024-11-11T23:54:34,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/fc385286f7474265a65d6a59a8285888 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/fc385286f7474265a65d6a59a8285888 2024-11-11T23:54:34,799 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/fc385286f7474265a65d6a59a8285888, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T23:54:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/2f09dd50c7ba4d688833ced7ba183001 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2f09dd50c7ba4d688833ced7ba183001 2024-11-11T23:54:34,804 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2f09dd50c7ba4d688833ced7ba183001, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T23:54:34,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/15802501efb4423486e996c6260b9c43 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/15802501efb4423486e996c6260b9c43 2024-11-11T23:54:34,809 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/15802501efb4423486e996c6260b9c43, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T23:54:34,810 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 1311b3c706ba19435848d682254b17d6 in 87ms, sequenceid=168, compaction requested=true 2024-11-11T23:54:34,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:34,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:34,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-11T23:54:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-11T23:54:34,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T23:54:34,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-11T23:54:34,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.0090 sec 2024-11-11T23:54:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T23:54:34,945 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:34,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-11T23:54:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T23:54:34,947 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:34,947 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:34,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:34,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:54:34,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:34,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:34,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:34,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:34,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:34,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:35,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/74ada8e0b5f9475cb1d0516a19ff9165 is 50, key is test_row_0/A:col10/1731369274971/Put/seqid=0 2024-11-11T23:54:35,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742075_1251 (size=14541) 2024-11-11T23:54:35,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/74ada8e0b5f9475cb1d0516a19ff9165 2024-11-11T23:54:35,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/de110b555d0f41d18251406fd7e62a39 is 50, key is test_row_0/B:col10/1731369274971/Put/seqid=0 2024-11-11T23:54:35,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742076_1252 (size=12151) 2024-11-11T23:54:35,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/de110b555d0f41d18251406fd7e62a39 2024-11-11T23:54:35,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/93fcd8dbef5d47668f4184d0e2132900 is 50, key is test_row_0/C:col10/1731369274971/Put/seqid=0 2024-11-11T23:54:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742077_1253 (size=12151) 2024-11-11T23:54:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T23:54:35,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369275061, max timestamp=1731369275081Num uniques 859; 2024-11-11T23:54:35,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369275061, max timestamp=1731369275081Num uniques 859; 2024-11-11T23:54:35,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:35,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:35,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369275061, max timestamp=1731369275081Num uniques 859; 2024-11-11T23:54:35,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:35,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:35,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:35,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369335172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:35,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T23:54:35,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369335284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:35,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:35,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:35,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/93fcd8dbef5d47668f4184d0e2132900 2024-11-11T23:54:35,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/74ada8e0b5f9475cb1d0516a19ff9165 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/74ada8e0b5f9475cb1d0516a19ff9165 2024-11-11T23:54:35,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/74ada8e0b5f9475cb1d0516a19ff9165, entries=200, sequenceid=178, filesize=14.2 K 2024-11-11T23:54:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/de110b555d0f41d18251406fd7e62a39 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/de110b555d0f41d18251406fd7e62a39 2024-11-11T23:54:35,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/de110b555d0f41d18251406fd7e62a39, entries=150, sequenceid=178, filesize=11.9 K 2024-11-11T23:54:35,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/93fcd8dbef5d47668f4184d0e2132900 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/93fcd8dbef5d47668f4184d0e2132900 2024-11-11T23:54:35,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/93fcd8dbef5d47668f4184d0e2132900, entries=150, sequenceid=178, filesize=11.9 K 2024-11-11T23:54:35,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1311b3c706ba19435848d682254b17d6 in 479ms, sequenceid=178, compaction requested=true 2024-11-11T23:54:35,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:35,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:35,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:35,471 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:35,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:35,471 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:35,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:35,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:35,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:35,472 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53624 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:35,472 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:35,472 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:35,472 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:35,472 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,472 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,472 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/52b005d9ff7e42628724a54b01b9c933, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/994c070fd5c1428880d753aa9f14317f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/fc385286f7474265a65d6a59a8285888, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/74ada8e0b5f9475cb1d0516a19ff9165] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=52.4 K 2024-11-11T23:54:35,472 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2d9f900350d1413193f0b0c2fe6598d5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/11c7102677b34a98920e099b7fdfee12, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2f09dd50c7ba4d688833ced7ba183001, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/de110b555d0f41d18251406fd7e62a39] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=47.7 K 2024-11-11T23:54:35,472 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b005d9ff7e42628724a54b01b9c933, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731369271739 2024-11-11T23:54:35,472 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d9f900350d1413193f0b0c2fe6598d5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731369271739 2024-11-11T23:54:35,473 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 11c7102677b34a98920e099b7fdfee12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369272405 2024-11-11T23:54:35,473 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 994c070fd5c1428880d753aa9f14317f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369272405 2024-11-11T23:54:35,473 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f09dd50c7ba4d688833ced7ba183001, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731369273790 2024-11-11T23:54:35,473 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc385286f7474265a65d6a59a8285888, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731369273790 2024-11-11T23:54:35,473 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting de110b555d0f41d18251406fd7e62a39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731369274971 2024-11-11T23:54:35,473 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ada8e0b5f9475cb1d0516a19ff9165, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731369274969 2024-11-11T23:54:35,481 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:35,481 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/890d13564bf748f380e3c38940b2b2f4 is 50, key is test_row_0/B:col10/1731369274971/Put/seqid=0 2024-11-11T23:54:35,482 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:35,482 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/c7f33abb4fcf4d8f9cecf8be654347a9 is 50, key is test_row_0/A:col10/1731369274971/Put/seqid=0 2024-11-11T23:54:35,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742078_1254 (size=12527) 2024-11-11T23:54:35,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742079_1255 (size=12527) 2024-11-11T23:54:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:35,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T23:54:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:54:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:54:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:54:35,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/e0429883f5674ce4ae54800bb217ad5a is 50, key is test_row_0/A:col10/1731369275171/Put/seqid=0 2024-11-11T23:54:35,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742080_1256 (size=14541) 2024-11-11T23:54:35,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369335536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:35,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:35,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:35,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:35,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T23:54:35,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369335644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:35,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:35,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:35,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369335854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:35,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:35,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:35,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:35,895 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/890d13564bf748f380e3c38940b2b2f4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/890d13564bf748f380e3c38940b2b2f4 2024-11-11T23:54:35,899 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into 890d13564bf748f380e3c38940b2b2f4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:35,899 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:35,899 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=12, startTime=1731369275471; duration=0sec 2024-11-11T23:54:35,899 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:35,899 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/c7f33abb4fcf4d8f9cecf8be654347a9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/c7f33abb4fcf4d8f9cecf8be654347a9 2024-11-11T23:54:35,899 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:35,899 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:35,901 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:35,901 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:35,901 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:35,902 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/506396762cc44f90a215db2af73f9578, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c93d53d214d445a4bba038bd4b8c852a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/15802501efb4423486e996c6260b9c43, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/93fcd8dbef5d47668f4184d0e2132900] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=47.7 K 2024-11-11T23:54:35,902 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 506396762cc44f90a215db2af73f9578, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731369271739 2024-11-11T23:54:35,902 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c93d53d214d445a4bba038bd4b8c852a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369272405 2024-11-11T23:54:35,902 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 15802501efb4423486e996c6260b9c43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731369273790 2024-11-11T23:54:35,903 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 93fcd8dbef5d47668f4184d0e2132900, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731369274971 2024-11-11T23:54:35,905 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into c7f33abb4fcf4d8f9cecf8be654347a9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:35,905 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:35,905 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=12, startTime=1731369275471; duration=0sec 2024-11-11T23:54:35,905 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:35,905 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:35,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/e0429883f5674ce4ae54800bb217ad5a 2024-11-11T23:54:35,911 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:35,912 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/64c1fe1b33cd4a0682ae6f201dd1a576 is 50, key is test_row_0/C:col10/1731369274971/Put/seqid=0 2024-11-11T23:54:35,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742081_1257 (size=12527) 2024-11-11T23:54:35,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/356fef595d3a4b11855efb984f4ea61a is 50, key is test_row_0/B:col10/1731369275171/Put/seqid=0 2024-11-11T23:54:35,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742082_1258 (size=12151) 2024-11-11T23:54:35,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/356fef595d3a4b11855efb984f4ea61a 2024-11-11T23:54:35,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/cc05ae80be494ed9b6ce2e7e69d20994 is 50, key is test_row_0/C:col10/1731369275171/Put/seqid=0 2024-11-11T23:54:35,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742083_1259 (size=12151) 2024-11-11T23:54:36,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:36,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T23:54:36,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369336164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:36,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:36,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,322 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/64c1fe1b33cd4a0682ae6f201dd1a576 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/64c1fe1b33cd4a0682ae6f201dd1a576 2024-11-11T23:54:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:36,328 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 64c1fe1b33cd4a0682ae6f201dd1a576(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:36,328 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:36,328 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=12, startTime=1731369275471; duration=0sec 2024-11-11T23:54:36,328 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:36,328 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:36,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/cc05ae80be494ed9b6ce2e7e69d20994 2024-11-11T23:54:36,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/e0429883f5674ce4ae54800bb217ad5a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e0429883f5674ce4ae54800bb217ad5a 2024-11-11T23:54:36,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e0429883f5674ce4ae54800bb217ad5a, entries=200, sequenceid=216, filesize=14.2 K 2024-11-11T23:54:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/356fef595d3a4b11855efb984f4ea61a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/356fef595d3a4b11855efb984f4ea61a 2024-11-11T23:54:36,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/356fef595d3a4b11855efb984f4ea61a, entries=150, sequenceid=216, filesize=11.9 K 2024-11-11T23:54:36,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/cc05ae80be494ed9b6ce2e7e69d20994 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cc05ae80be494ed9b6ce2e7e69d20994 2024-11-11T23:54:36,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cc05ae80be494ed9b6ce2e7e69d20994, entries=150, sequenceid=216, filesize=11.9 K 2024-11-11T23:54:36,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 1311b3c706ba19435848d682254b17d6 in 856ms, sequenceid=216, compaction requested=false 2024-11-11T23:54:36,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:36,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T23:54:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:36,476 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T23:54:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/0a8af059ecd343f49ef1cfc207a597b8 is 50, key is test_row_0/A:col10/1731369275535/Put/seqid=0 2024-11-11T23:54:36,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742084_1260 (size=12151) 2024-11-11T23:54:36,484 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/0a8af059ecd343f49ef1cfc207a597b8 2024-11-11T23:54:36,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/8006f53d753344399f6cb6749bae1eaf is 50, key is test_row_0/B:col10/1731369275535/Put/seqid=0 2024-11-11T23:54:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742085_1261 (size=12151) 2024-11-11T23:54:36,504 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/8006f53d753344399f6cb6749bae1eaf 2024-11-11T23:54:36,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c59f42507722481f9b597b2852dcbaa0 is 50, key is test_row_0/C:col10/1731369275535/Put/seqid=0 2024-11-11T23:54:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742086_1262 (size=12151) 2024-11-11T23:54:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:36,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:36,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369276605, max timestamp=1731369276612Num uniques 859; 2024-11-11T23:54:36,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369336610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369336623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369336627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369336630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369336674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369336724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369336734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369336744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369336745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,925 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c59f42507722481f9b597b2852dcbaa0 2024-11-11T23:54:36,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/0a8af059ecd343f49ef1cfc207a597b8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/0a8af059ecd343f49ef1cfc207a597b8 2024-11-11T23:54:36,933 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/0a8af059ecd343f49ef1cfc207a597b8, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T23:54:36,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/8006f53d753344399f6cb6749bae1eaf as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8006f53d753344399f6cb6749bae1eaf 2024-11-11T23:54:36,938 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8006f53d753344399f6cb6749bae1eaf, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T23:54:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c59f42507722481f9b597b2852dcbaa0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c59f42507722481f9b597b2852dcbaa0 2024-11-11T23:54:36,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369336934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c59f42507722481f9b597b2852dcbaa0, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T23:54:36,944 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=136.42 KB/139690 for 1311b3c706ba19435848d682254b17d6 in 468ms, sequenceid=228, compaction requested=true 2024-11-11T23:54:36,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:36,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:36,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-11T23:54:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-11T23:54:36,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T23:54:36,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9990 sec 2024-11-11T23:54:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:36,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=143.13 KB heapSize=421.48 KB 2024-11-11T23:54:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:36,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.0050 sec 2024-11-11T23:54:36,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1f5a2195bd1742fbb314035de46fe07d is 50, key is test_row_0/A:col10/1731369276620/Put/seqid=0 2024-11-11T23:54:36,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742087_1263 (size=14591) 2024-11-11T23:54:36,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369336985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369336990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369336991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T23:54:37,094 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:37,095 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T23:54:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T23:54:37,096 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:37,097 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:37,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:37,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369337094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369337104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369337104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T23:54:37,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369337245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T23:54:37,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:37,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:37,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:37,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:37,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:37,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369337304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369337314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369337314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1f5a2195bd1742fbb314035de46fe07d 2024-11-11T23:54:37,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/9ad21d702612434ea93d0dc922478364 is 50, key is test_row_0/B:col10/1731369276620/Put/seqid=0 2024-11-11T23:54:37,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742088_1264 (size=12201) 2024-11-11T23:54:37,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/9ad21d702612434ea93d0dc922478364 2024-11-11T23:54:37,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c6154ed264834268a596ca27750b7e8f is 50, key is test_row_0/C:col10/1731369276620/Put/seqid=0 2024-11-11T23:54:37,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T23:54:37,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:37,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:37,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:37,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:37,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:37,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742089_1265 (size=12201) 2024-11-11T23:54:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T23:54:37,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c6154ed264834268a596ca27750b7e8f 2024-11-11T23:54:37,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1f5a2195bd1742fbb314035de46fe07d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1f5a2195bd1742fbb314035de46fe07d 2024-11-11T23:54:37,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1f5a2195bd1742fbb314035de46fe07d, entries=200, sequenceid=257, filesize=14.2 K 2024-11-11T23:54:37,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/9ad21d702612434ea93d0dc922478364 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/9ad21d702612434ea93d0dc922478364 2024-11-11T23:54:37,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/9ad21d702612434ea93d0dc922478364, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T23:54:37,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/c6154ed264834268a596ca27750b7e8f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c6154ed264834268a596ca27750b7e8f 2024-11-11T23:54:37,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c6154ed264834268a596ca27750b7e8f, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T23:54:37,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~421.44 KB/431552, currentSize=33.54 KB/34350 for 1311b3c706ba19435848d682254b17d6 in 493ms, sequenceid=257, compaction requested=true 2024-11-11T23:54:37,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:37,442 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:37,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:37,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:37,443 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:37,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:37,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:37,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:37,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:37,444 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:37,444 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:37,444 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:37,444 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/c7f33abb4fcf4d8f9cecf8be654347a9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e0429883f5674ce4ae54800bb217ad5a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/0a8af059ecd343f49ef1cfc207a597b8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1f5a2195bd1742fbb314035de46fe07d] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=52.5 K 2024-11-11T23:54:37,444 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49030 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:37,444 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:37,444 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:37,444 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/890d13564bf748f380e3c38940b2b2f4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/356fef595d3a4b11855efb984f4ea61a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8006f53d753344399f6cb6749bae1eaf, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/9ad21d702612434ea93d0dc922478364] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=47.9 K 2024-11-11T23:54:37,444 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7f33abb4fcf4d8f9cecf8be654347a9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731369274971 2024-11-11T23:54:37,445 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 890d13564bf748f380e3c38940b2b2f4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731369274971 2024-11-11T23:54:37,445 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0429883f5674ce4ae54800bb217ad5a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731369275160 2024-11-11T23:54:37,445 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 356fef595d3a4b11855efb984f4ea61a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731369275165 2024-11-11T23:54:37,446 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a8af059ecd343f49ef1cfc207a597b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731369275522 2024-11-11T23:54:37,446 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8006f53d753344399f6cb6749bae1eaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731369275522 2024-11-11T23:54:37,446 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad21d702612434ea93d0dc922478364, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731369276620 2024-11-11T23:54:37,446 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f5a2195bd1742fbb314035de46fe07d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731369276620 2024-11-11T23:54:37,462 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:37,462 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/8870a3599da54758ad5763d6aeeb6761 is 50, key is test_row_0/B:col10/1731369276620/Put/seqid=0 2024-11-11T23:54:37,465 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:37,465 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/3c1cb762c95c4a1ca5deb2ddc9ccafdd is 50, key is test_row_0/A:col10/1731369276620/Put/seqid=0 2024-11-11T23:54:37,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742091_1267 (size=12713) 2024-11-11T23:54:37,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742090_1266 (size=12713) 2024-11-11T23:54:37,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T23:54:37,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:37,556 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T23:54:37,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:37,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:37,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:37,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:37,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:37,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:37,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/06315ee34c734429a043da7afc11be5d is 50, key is test_row_0/A:col10/1731369276960/Put/seqid=0 2024-11-11T23:54:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742092_1268 (size=12301) 2024-11-11T23:54:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:37,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:37,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369277646, max timestamp=1731369277657Num uniques 859; 2024-11-11T23:54:37,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:37,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369277646, max timestamp=1731369277657Num uniques 859; 2024-11-11T23:54:37,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369277646, max timestamp=1731369277657Num uniques 859; 2024-11-11T23:54:37,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:37,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:54:37,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369277688, max timestamp=1731369277694Num uniques 1018; 2024-11-11T23:54:37,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:54:37,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369277688, max timestamp=1731369277694Num uniques 1018; 2024-11-11T23:54:37,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:54:37,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369277688, max timestamp=1731369277694Num uniques 1018; 2024-11-11T23:54:37,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:54:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T23:54:37,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731369277715, max timestamp=1731369277723Num uniques 1027; 2024-11-11T23:54:37,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T23:54:37,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731369277715, max timestamp=1731369277723Num uniques 1027; 2024-11-11T23:54:37,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T23:54:37,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731369277715, max timestamp=1731369277723Num uniques 1027; 2024-11-11T23:54:37,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T23:54:37,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731369277723, max timestamp=1731369277756Num uniques 1086; 2024-11-11T23:54:37,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-11T23:54:37,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731369277723, max timestamp=1731369277756Num uniques 1086; 2024-11-11T23:54:37,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731369277723, max timestamp=1731369277756Num uniques 1086; 2024-11-11T23:54:37,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-11T23:54:37,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-11T23:54:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369337782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369337783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369337784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369337786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369337789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,876 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/3c1cb762c95c4a1ca5deb2ddc9ccafdd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3c1cb762c95c4a1ca5deb2ddc9ccafdd 2024-11-11T23:54:37,881 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/8870a3599da54758ad5763d6aeeb6761 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8870a3599da54758ad5763d6aeeb6761 2024-11-11T23:54:37,883 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into 3c1cb762c95c4a1ca5deb2ddc9ccafdd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:37,883 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:37,883 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=12, startTime=1731369277442; duration=0sec 2024-11-11T23:54:37,883 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:37,883 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:37,883 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:37,884 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49030 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:37,884 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:37,884 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:37,884 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/64c1fe1b33cd4a0682ae6f201dd1a576, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cc05ae80be494ed9b6ce2e7e69d20994, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c59f42507722481f9b597b2852dcbaa0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c6154ed264834268a596ca27750b7e8f] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=47.9 K 2024-11-11T23:54:37,884 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c1fe1b33cd4a0682ae6f201dd1a576, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731369274971 2024-11-11T23:54:37,885 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc05ae80be494ed9b6ce2e7e69d20994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731369275165 2024-11-11T23:54:37,885 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c59f42507722481f9b597b2852dcbaa0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731369275522 2024-11-11T23:54:37,885 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6154ed264834268a596ca27750b7e8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731369276620 2024-11-11T23:54:37,887 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into 8870a3599da54758ad5763d6aeeb6761(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:37,887 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:37,887 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=12, startTime=1731369277443; duration=0sec 2024-11-11T23:54:37,887 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:37,887 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:37,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369337894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,895 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,896 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/40d7c0a703cc451184fbc5f733baacea is 50, key is test_row_0/C:col10/1731369276620/Put/seqid=0 2024-11-11T23:54:37,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369337894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742093_1269 (size=12713) 2024-11-11T23:54:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369337905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369337906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369337906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:37,915 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/40d7c0a703cc451184fbc5f733baacea as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/40d7c0a703cc451184fbc5f733baacea 2024-11-11T23:54:37,922 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 40d7c0a703cc451184fbc5f733baacea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:37,923 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:37,923 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=12, startTime=1731369277443; duration=0sec 2024-11-11T23:54:37,923 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:37,923 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:37,964 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/06315ee34c734429a043da7afc11be5d 2024-11-11T23:54:37,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/25d4131d16f54e219641d97261e4bc5c is 50, key is test_row_0/B:col10/1731369276960/Put/seqid=0 2024-11-11T23:54:37,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742094_1270 (size=12301) 2024-11-11T23:54:37,974 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/25d4131d16f54e219641d97261e4bc5c 2024-11-11T23:54:37,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/8edd7a3ad3f6431fbbabf211547f91ad is 50, key is test_row_0/C:col10/1731369276960/Put/seqid=0 2024-11-11T23:54:37,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742095_1271 (size=12301) 2024-11-11T23:54:38,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369338104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369338104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369338115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369338115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369338124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T23:54:38,385 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/8edd7a3ad3f6431fbbabf211547f91ad 2024-11-11T23:54:38,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/06315ee34c734429a043da7afc11be5d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/06315ee34c734429a043da7afc11be5d 2024-11-11T23:54:38,393 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/06315ee34c734429a043da7afc11be5d, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T23:54:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/25d4131d16f54e219641d97261e4bc5c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/25d4131d16f54e219641d97261e4bc5c 2024-11-11T23:54:38,398 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/25d4131d16f54e219641d97261e4bc5c, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T23:54:38,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/8edd7a3ad3f6431fbbabf211547f91ad as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/8edd7a3ad3f6431fbbabf211547f91ad 2024-11-11T23:54:38,402 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/8edd7a3ad3f6431fbbabf211547f91ad, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T23:54:38,403 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 1311b3c706ba19435848d682254b17d6 in 847ms, sequenceid=265, compaction requested=false 2024-11-11T23:54:38,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:38,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:38,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T23:54:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T23:54:38,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T23:54:38,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3070 sec 2024-11-11T23:54:38,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.3110 sec 2024-11-11T23:54:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:38,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=161.02 KB heapSize=448.78 KB 2024-11-11T23:54:38,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:38,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:54:38,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:38,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:54:38,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:38,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:54:38,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/77281ab4824b4ea1887ffaa39a4c1236 is 50, key is test_row_0/A:col10/1731369277784/Put/seqid=0 2024-11-11T23:54:38,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742096_1272 (size=14741) 2024-11-11T23:54:38,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369338423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369338425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369338425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369338428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369338434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369338534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369338534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369338745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369338745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:38,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/77281ab4824b4ea1887ffaa39a4c1236 2024-11-11T23:54:38,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/aabefd1b09cd49b8ab9c7197f3549025 is 50, key is test_row_0/B:col10/1731369277784/Put/seqid=0 2024-11-11T23:54:38,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742097_1273 (size=12301) 2024-11-11T23:54:38,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/aabefd1b09cd49b8ab9c7197f3549025 2024-11-11T23:54:38,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/04b378d6961a4a4fb549b333d3f619ca is 50, key is test_row_0/C:col10/1731369277784/Put/seqid=0 2024-11-11T23:54:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742098_1274 (size=12301) 2024-11-11T23:54:38,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/04b378d6961a4a4fb549b333d3f619ca 2024-11-11T23:54:38,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/77281ab4824b4ea1887ffaa39a4c1236 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/77281ab4824b4ea1887ffaa39a4c1236 2024-11-11T23:54:38,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/77281ab4824b4ea1887ffaa39a4c1236, entries=200, sequenceid=357, filesize=14.4 K 2024-11-11T23:54:38,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/aabefd1b09cd49b8ab9c7197f3549025 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/aabefd1b09cd49b8ab9c7197f3549025 2024-11-11T23:54:38,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/aabefd1b09cd49b8ab9c7197f3549025, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T23:54:38,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/04b378d6961a4a4fb549b333d3f619ca as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/04b378d6961a4a4fb549b333d3f619ca 2024-11-11T23:54:38,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/04b378d6961a4a4fb549b333d3f619ca, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T23:54:38,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~448.73 KB/459504, currentSize=26.84 KB/27480 for 1311b3c706ba19435848d682254b17d6 in 495ms, sequenceid=357, compaction requested=true 2024-11-11T23:54:38,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:38,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:38,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:38,911 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:38,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:38,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:38,911 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:38,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:38,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:38,912 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:38,912 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:38,912 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:38,912 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8870a3599da54758ad5763d6aeeb6761, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/25d4131d16f54e219641d97261e4bc5c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/aabefd1b09cd49b8ab9c7197f3549025] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=36.4 K 2024-11-11T23:54:38,913 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8870a3599da54758ad5763d6aeeb6761, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731369276620 2024-11-11T23:54:38,913 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:38,913 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:38,913 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:38,913 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3c1cb762c95c4a1ca5deb2ddc9ccafdd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/06315ee34c734429a043da7afc11be5d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/77281ab4824b4ea1887ffaa39a4c1236] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=38.8 K 2024-11-11T23:54:38,914 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1cb762c95c4a1ca5deb2ddc9ccafdd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731369276620 2024-11-11T23:54:38,914 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d4131d16f54e219641d97261e4bc5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369276960 2024-11-11T23:54:38,914 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting aabefd1b09cd49b8ab9c7197f3549025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369277784 2024-11-11T23:54:38,914 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06315ee34c734429a043da7afc11be5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369276960 2024-11-11T23:54:38,914 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77281ab4824b4ea1887ffaa39a4c1236, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369277781 2024-11-11T23:54:38,929 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:38,930 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/687175a5a247440b8343f66655106064 is 50, key is test_row_0/B:col10/1731369277784/Put/seqid=0 2024-11-11T23:54:38,938 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:38,938 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/91f5cddf535a49eabd281b8f4f8af792 is 50, key is test_row_0/A:col10/1731369277784/Put/seqid=0 2024-11-11T23:54:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:38,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:54:38,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:38,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:38,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:38,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:38,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:38,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:38,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742099_1275 (size=12915) 2024-11-11T23:54:38,969 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/687175a5a247440b8343f66655106064 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/687175a5a247440b8343f66655106064 2024-11-11T23:54:38,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/caa90875b79643079efb5e6627c91362 is 50, key is test_row_1/A:col10/1731369278937/Put/seqid=0 2024-11-11T23:54:38,979 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into 687175a5a247440b8343f66655106064(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:38,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:38,979 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=13, startTime=1731369278911; duration=0sec 2024-11-11T23:54:38,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:38,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:38,980 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:38,982 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:38,982 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:38,982 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:38,982 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/40d7c0a703cc451184fbc5f733baacea, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/8edd7a3ad3f6431fbbabf211547f91ad, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/04b378d6961a4a4fb549b333d3f619ca] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=36.4 K 2024-11-11T23:54:38,983 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d7c0a703cc451184fbc5f733baacea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731369276620 2024-11-11T23:54:38,984 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8edd7a3ad3f6431fbbabf211547f91ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731369276960 2024-11-11T23:54:38,984 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b378d6961a4a4fb549b333d3f619ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369277784 2024-11-11T23:54:38,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742100_1276 (size=12915) 2024-11-11T23:54:38,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369278979, max timestamp=1731369278991Num uniques 859; 2024-11-11T23:54:38,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369278979, max timestamp=1731369278991Num uniques 859; 2024-11-11T23:54:38,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369278979, max timestamp=1731369278991Num uniques 859; 2024-11-11T23:54:38,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:54:38,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:54:38,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:54:39,001 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/91f5cddf535a49eabd281b8f4f8af792 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/91f5cddf535a49eabd281b8f4f8af792 2024-11-11T23:54:39,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742101_1277 (size=12297) 2024-11-11T23:54:39,008 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into 91f5cddf535a49eabd281b8f4f8af792(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:39,008 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:39,008 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=13, startTime=1731369278911; duration=0sec 2024-11-11T23:54:39,009 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:39,009 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:39,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/caa90875b79643079efb5e6627c91362 2024-11-11T23:54:39,015 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:39,018 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/2cbe91b04d7f40528b999da2bc14b667 is 50, key is test_row_0/C:col10/1731369277784/Put/seqid=0 2024-11-11T23:54:39,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6856f3d81ac940a0980bd4a213d62e57 is 50, key is test_row_1/B:col10/1731369278937/Put/seqid=0 2024-11-11T23:54:39,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742102_1278 (size=12915) 2024-11-11T23:54:39,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369339035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369339036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369339043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369339055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369339055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742103_1279 (size=9857) 2024-11-11T23:54:39,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6856f3d81ac940a0980bd4a213d62e57 2024-11-11T23:54:39,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/94902816f29c4f7988ddaf3a429ff347 is 50, key is test_row_1/C:col10/1731369278937/Put/seqid=0 2024-11-11T23:54:39,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742104_1280 (size=9857) 2024-11-11T23:54:39,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369339155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369339155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369339155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T23:54:39,245 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:39,245 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T23:54:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T23:54:39,247 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:39,247 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:39,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T23:54:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369339365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369339365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369339365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T23:54:39,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:39,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:39,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:39,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:39,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:39,444 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/2cbe91b04d7f40528b999da2bc14b667 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2cbe91b04d7f40528b999da2bc14b667 2024-11-11T23:54:39,447 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 2cbe91b04d7f40528b999da2bc14b667(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:39,447 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:39,447 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=13, startTime=1731369278911; duration=0sec 2024-11-11T23:54:39,448 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:39,448 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:39,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/94902816f29c4f7988ddaf3a429ff347 2024-11-11T23:54:39,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/caa90875b79643079efb5e6627c91362 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/caa90875b79643079efb5e6627c91362 2024-11-11T23:54:39,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/caa90875b79643079efb5e6627c91362, entries=150, sequenceid=368, filesize=12.0 K 2024-11-11T23:54:39,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6856f3d81ac940a0980bd4a213d62e57 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6856f3d81ac940a0980bd4a213d62e57 2024-11-11T23:54:39,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6856f3d81ac940a0980bd4a213d62e57, entries=100, sequenceid=368, filesize=9.6 K 2024-11-11T23:54:39,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/94902816f29c4f7988ddaf3a429ff347 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/94902816f29c4f7988ddaf3a429ff347 2024-11-11T23:54:39,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/94902816f29c4f7988ddaf3a429ff347, entries=100, sequenceid=368, filesize=9.6 K 2024-11-11T23:54:39,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1311b3c706ba19435848d682254b17d6 in 568ms, sequenceid=368, compaction requested=false 2024-11-11T23:54:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:39,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T23:54:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:39,553 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T23:54:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:39,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:39,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:39,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:39,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1cf87a858c06494db516a1e627ef7289 is 50, key is test_row_0/A:col10/1731369279029/Put/seqid=0 2024-11-11T23:54:39,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742105_1281 (size=12301) 2024-11-11T23:54:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T23:54:39,565 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1cf87a858c06494db516a1e627ef7289 2024-11-11T23:54:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:39,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:39,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/4d7397cfec8843a790417172981ce80a is 50, key is test_row_0/B:col10/1731369279029/Put/seqid=0 2024-11-11T23:54:39,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369339592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369339592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742106_1282 (size=12301) 2024-11-11T23:54:39,603 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/4d7397cfec8843a790417172981ce80a 2024-11-11T23:54:39,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b778f3f24aea4445938e30fbd79ea7ab is 50, key is test_row_0/C:col10/1731369279029/Put/seqid=0 2024-11-11T23:54:39,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742107_1283 (size=12301) 2024-11-11T23:54:39,621 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b778f3f24aea4445938e30fbd79ea7ab 2024-11-11T23:54:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1cf87a858c06494db516a1e627ef7289 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1cf87a858c06494db516a1e627ef7289 2024-11-11T23:54:39,631 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1cf87a858c06494db516a1e627ef7289, entries=150, sequenceid=406, filesize=12.0 K 2024-11-11T23:54:39,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/4d7397cfec8843a790417172981ce80a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4d7397cfec8843a790417172981ce80a 2024-11-11T23:54:39,639 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4d7397cfec8843a790417172981ce80a, entries=150, sequenceid=406, filesize=12.0 K 2024-11-11T23:54:39,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b778f3f24aea4445938e30fbd79ea7ab as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b778f3f24aea4445938e30fbd79ea7ab 2024-11-11T23:54:39,644 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b778f3f24aea4445938e30fbd79ea7ab, entries=150, sequenceid=406, filesize=12.0 K 2024-11-11T23:54:39,645 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 1311b3c706ba19435848d682254b17d6 in 92ms, sequenceid=406, compaction requested=true 2024-11-11T23:54:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:39,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T23:54:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T23:54:39,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T23:54:39,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 400 msec 2024-11-11T23:54:39,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 404 msec 2024-11-11T23:54:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:39,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T23:54:39,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:39,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:39,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:39,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:39,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:39,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:39,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/3ed50d43cdee4cecb832011787a27945 is 50, key is test_row_0/A:col10/1731369279584/Put/seqid=0 2024-11-11T23:54:39,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742108_1284 (size=12301) 2024-11-11T23:54:39,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369339717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369339747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369339747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369339747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369339748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369339855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369339865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369339865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369339865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369339865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T23:54:39,874 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:39,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T23:54:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T23:54:39,876 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:39,877 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:39,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T23:54:40,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T23:54:40,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:40,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369340065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369340074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369340074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369340074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369340074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/3ed50d43cdee4cecb832011787a27945 2024-11-11T23:54:40,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6ee0f6f23908430fb5fbce6438ba2317 is 50, key is test_row_0/B:col10/1731369279584/Put/seqid=0 2024-11-11T23:54:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742109_1285 (size=12301) 2024-11-11T23:54:40,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6ee0f6f23908430fb5fbce6438ba2317 2024-11-11T23:54:40,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/2981b6a6432f4bd38636d0e243dfea33 is 50, key is test_row_0/C:col10/1731369279584/Put/seqid=0 2024-11-11T23:54:40,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742110_1286 (size=12301) 2024-11-11T23:54:40,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T23:54:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T23:54:40,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T23:54:40,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:40,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369340375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369340384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369340384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369340384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369340384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T23:54:40,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:40,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T23:54:40,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/2981b6a6432f4bd38636d0e243dfea33 2024-11-11T23:54:40,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/3ed50d43cdee4cecb832011787a27945 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3ed50d43cdee4cecb832011787a27945 2024-11-11T23:54:40,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3ed50d43cdee4cecb832011787a27945, entries=150, sequenceid=418, filesize=12.0 K 2024-11-11T23:54:40,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6ee0f6f23908430fb5fbce6438ba2317 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6ee0f6f23908430fb5fbce6438ba2317 2024-11-11T23:54:40,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6ee0f6f23908430fb5fbce6438ba2317, entries=150, sequenceid=418, filesize=12.0 K 2024-11-11T23:54:40,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/2981b6a6432f4bd38636d0e243dfea33 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2981b6a6432f4bd38636d0e243dfea33 2024-11-11T23:54:40,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2981b6a6432f4bd38636d0e243dfea33, entries=150, sequenceid=418, filesize=12.0 K 2024-11-11T23:54:40,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1311b3c706ba19435848d682254b17d6 in 871ms, sequenceid=418, compaction requested=true 2024-11-11T23:54:40,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:40,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:40,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:40,548 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:40,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:40,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:40,548 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:40,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:40,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:40,549 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:40,550 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:40,550 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,550 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49814 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:40,550 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:40,550 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/687175a5a247440b8343f66655106064, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6856f3d81ac940a0980bd4a213d62e57, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4d7397cfec8843a790417172981ce80a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6ee0f6f23908430fb5fbce6438ba2317] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=46.3 K 2024-11-11T23:54:40,550 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,550 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/91f5cddf535a49eabd281b8f4f8af792, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/caa90875b79643079efb5e6627c91362, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1cf87a858c06494db516a1e627ef7289, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3ed50d43cdee4cecb832011787a27945] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=48.6 K 2024-11-11T23:54:40,550 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f5cddf535a49eabd281b8f4f8af792, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369277784 2024-11-11T23:54:40,550 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 687175a5a247440b8343f66655106064, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369277784 2024-11-11T23:54:40,551 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6856f3d81ac940a0980bd4a213d62e57, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731369278937 2024-11-11T23:54:40,551 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa90875b79643079efb5e6627c91362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731369278937 2024-11-11T23:54:40,551 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d7397cfec8843a790417172981ce80a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731369279020 2024-11-11T23:54:40,551 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf87a858c06494db516a1e627ef7289, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731369279020 2024-11-11T23:54:40,551 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee0f6f23908430fb5fbce6438ba2317, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731369279584 2024-11-11T23:54:40,551 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ed50d43cdee4cecb832011787a27945, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731369279584 2024-11-11T23:54:40,558 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:40,559 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/7c61117ce7a645d3ae349f0e2909e9fe is 50, key is test_row_0/A:col10/1731369279584/Put/seqid=0 2024-11-11T23:54:40,559 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:40,559 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/bf5196c20e7e4ff083d9e417b9a9ada1 is 50, key is test_row_0/B:col10/1731369279584/Put/seqid=0 2024-11-11T23:54:40,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742112_1288 (size=13051) 2024-11-11T23:54:40,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742111_1287 (size=13051) 2024-11-11T23:54:40,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T23:54:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,640 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T23:54:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:40,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/28ddb97b5ea345cf8909383f4d5a34a8 is 50, key is test_row_0/A:col10/1731369279746/Put/seqid=0 2024-11-11T23:54:40,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742113_1289 (size=12301) 2024-11-11T23:54:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:40,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:40,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369340900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369340903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369340904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369340905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369340907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:40,973 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/7c61117ce7a645d3ae349f0e2909e9fe as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/7c61117ce7a645d3ae349f0e2909e9fe 2024-11-11T23:54:40,973 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/bf5196c20e7e4ff083d9e417b9a9ada1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/bf5196c20e7e4ff083d9e417b9a9ada1 2024-11-11T23:54:40,979 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into bf5196c20e7e4ff083d9e417b9a9ada1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:40,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:40,979 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=12, startTime=1731369280548; duration=0sec 2024-11-11T23:54:40,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:40,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:40,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:40,980 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into 7c61117ce7a645d3ae349f0e2909e9fe(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:40,980 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:40,980 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=12, startTime=1731369280548; duration=0sec 2024-11-11T23:54:40,980 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:40,981 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:40,982 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:40,982 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:40,982 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:40,982 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2cbe91b04d7f40528b999da2bc14b667, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/94902816f29c4f7988ddaf3a429ff347, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b778f3f24aea4445938e30fbd79ea7ab, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2981b6a6432f4bd38636d0e243dfea33] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=46.3 K 2024-11-11T23:54:40,982 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cbe91b04d7f40528b999da2bc14b667, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369277784 2024-11-11T23:54:40,983 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 94902816f29c4f7988ddaf3a429ff347, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731369278937 2024-11-11T23:54:40,983 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b778f3f24aea4445938e30fbd79ea7ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731369279020 2024-11-11T23:54:40,983 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2981b6a6432f4bd38636d0e243dfea33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731369279584 2024-11-11T23:54:40,990 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:40,991 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/0a326de6ce4749f797a904aedf1ef780 is 50, key is test_row_0/C:col10/1731369279584/Put/seqid=0 2024-11-11T23:54:41,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742114_1290 (size=13051) 2024-11-11T23:54:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T23:54:41,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369341015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369341015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369341015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369341015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,048 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/28ddb97b5ea345cf8909383f4d5a34a8 2024-11-11T23:54:41,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/0a0fd247176042268362124ee0823e16 is 50, key is test_row_0/B:col10/1731369279746/Put/seqid=0 2024-11-11T23:54:41,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742115_1291 (size=12301) 2024-11-11T23:54:41,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369341225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369341225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369341225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369341225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,407 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/0a326de6ce4749f797a904aedf1ef780 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0a326de6ce4749f797a904aedf1ef780 2024-11-11T23:54:41,411 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 0a326de6ce4749f797a904aedf1ef780(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:41,411 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:41,411 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=12, startTime=1731369280548; duration=0sec 2024-11-11T23:54:41,412 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:41,412 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:41,458 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/0a0fd247176042268362124ee0823e16 2024-11-11T23:54:41,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/20f2be5d29364c12a94f2e890a16da2f is 50, key is test_row_0/C:col10/1731369279746/Put/seqid=0 2024-11-11T23:54:41,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742116_1292 (size=12301) 2024-11-11T23:54:41,470 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/20f2be5d29364c12a94f2e890a16da2f 2024-11-11T23:54:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/28ddb97b5ea345cf8909383f4d5a34a8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/28ddb97b5ea345cf8909383f4d5a34a8 2024-11-11T23:54:41,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/28ddb97b5ea345cf8909383f4d5a34a8, entries=150, sequenceid=438, filesize=12.0 K 2024-11-11T23:54:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/0a0fd247176042268362124ee0823e16 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/0a0fd247176042268362124ee0823e16 2024-11-11T23:54:41,484 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/0a0fd247176042268362124ee0823e16, entries=150, sequenceid=438, filesize=12.0 K 2024-11-11T23:54:41,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/20f2be5d29364c12a94f2e890a16da2f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/20f2be5d29364c12a94f2e890a16da2f 2024-11-11T23:54:41,488 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/20f2be5d29364c12a94f2e890a16da2f, entries=150, sequenceid=438, filesize=12.0 K 2024-11-11T23:54:41,489 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1311b3c706ba19435848d682254b17d6 in 849ms, sequenceid=438, compaction requested=false 2024-11-11T23:54:41,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:41,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:41,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T23:54:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T23:54:41,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T23:54:41,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6130 sec 2024-11-11T23:54:41,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6170 sec 2024-11-11T23:54:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:41,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T23:54:41,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:41,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:41,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:41,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:41,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:41,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:41,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1dceb6ef83c84782a16f219b909ce4f5 is 50, key is test_row_0/A:col10/1731369281536/Put/seqid=0 2024-11-11T23:54:41,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742117_1293 (size=12301) 2024-11-11T23:54:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369341553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369341554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369341555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369341664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369341664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369341664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369341664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369341875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369341875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369341875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369341875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369341915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:41,918 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 1311b3c706ba19435848d682254b17d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:54:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:41,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1dceb6ef83c84782a16f219b909ce4f5 2024-11-11T23:54:41,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ccba75b7808f4429882382c3bedb07e0 is 50, key is test_row_0/B:col10/1731369281536/Put/seqid=0 2024-11-11T23:54:41,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742118_1294 (size=12301) 2024-11-11T23:54:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T23:54:42,025 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:42,025 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T23:54:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T23:54:42,027 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:42,027 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:42,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T23:54:42,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T23:54:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:42,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369342184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:42,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369342184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:42,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369342184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369342184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:42,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T23:54:42,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T23:54:42,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ccba75b7808f4429882382c3bedb07e0 2024-11-11T23:54:42,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/cf13556f99ef4b77b318b774b0ff8857 is 50, key is test_row_0/C:col10/1731369281536/Put/seqid=0 2024-11-11T23:54:42,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742119_1295 (size=12301) 2024-11-11T23:54:42,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T23:54:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T23:54:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T23:54:42,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369342694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369342694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369342695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:42,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369342695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:42,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T23:54:42,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:42,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:42,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/cf13556f99ef4b77b318b774b0ff8857 2024-11-11T23:54:42,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/1dceb6ef83c84782a16f219b909ce4f5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1dceb6ef83c84782a16f219b909ce4f5 2024-11-11T23:54:42,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1dceb6ef83c84782a16f219b909ce4f5, entries=150, sequenceid=455, filesize=12.0 K 2024-11-11T23:54:42,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ccba75b7808f4429882382c3bedb07e0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ccba75b7808f4429882382c3bedb07e0 2024-11-11T23:54:42,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ccba75b7808f4429882382c3bedb07e0, entries=150, sequenceid=455, filesize=12.0 K 2024-11-11T23:54:42,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/cf13556f99ef4b77b318b774b0ff8857 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cf13556f99ef4b77b318b774b0ff8857 2024-11-11T23:54:42,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cf13556f99ef4b77b318b774b0ff8857, entries=150, sequenceid=455, filesize=12.0 K 2024-11-11T23:54:42,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1311b3c706ba19435848d682254b17d6 in 1277ms, sequenceid=455, compaction requested=true 2024-11-11T23:54:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:42,814 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:42,814 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:42,815 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:42,815 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:42,815 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:42,815 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:42,815 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,815 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,815 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/bf5196c20e7e4ff083d9e417b9a9ada1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/0a0fd247176042268362124ee0823e16, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ccba75b7808f4429882382c3bedb07e0] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=36.8 K 2024-11-11T23:54:42,815 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/7c61117ce7a645d3ae349f0e2909e9fe, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/28ddb97b5ea345cf8909383f4d5a34a8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1dceb6ef83c84782a16f219b909ce4f5] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=36.8 K 2024-11-11T23:54:42,816 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5196c20e7e4ff083d9e417b9a9ada1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731369279584 2024-11-11T23:54:42,816 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c61117ce7a645d3ae349f0e2909e9fe, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731369279584 2024-11-11T23:54:42,816 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a0fd247176042268362124ee0823e16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731369279712 2024-11-11T23:54:42,816 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28ddb97b5ea345cf8909383f4d5a34a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731369279712 2024-11-11T23:54:42,816 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ccba75b7808f4429882382c3bedb07e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731369280899 2024-11-11T23:54:42,816 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dceb6ef83c84782a16f219b909ce4f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731369280899 2024-11-11T23:54:42,826 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:42,827 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/d743d484db354ac7b07f41a9bf48999d is 50, key is test_row_0/A:col10/1731369281536/Put/seqid=0 2024-11-11T23:54:42,829 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:42,830 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/14f1b2764317489199814c6b4111433b is 50, key is test_row_0/B:col10/1731369281536/Put/seqid=0 2024-11-11T23:54:42,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742120_1296 (size=13153) 2024-11-11T23:54:42,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742121_1297 (size=13153) 2024-11-11T23:54:42,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T23:54:42,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:42,944 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T23:54:42,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:42,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:42,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:42,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:42,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:42,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:42,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/8ceec1eb3ff7442e8118a96385313b95 is 50, key is test_row_0/A:col10/1731369281546/Put/seqid=0 2024-11-11T23:54:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742122_1298 (size=12301) 2024-11-11T23:54:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T23:54:43,241 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/d743d484db354ac7b07f41a9bf48999d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/d743d484db354ac7b07f41a9bf48999d 2024-11-11T23:54:43,248 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into d743d484db354ac7b07f41a9bf48999d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:43,248 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:43,248 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=13, startTime=1731369282814; duration=0sec 2024-11-11T23:54:43,248 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:43,248 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:43,248 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:43,249 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/14f1b2764317489199814c6b4111433b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/14f1b2764317489199814c6b4111433b 2024-11-11T23:54:43,250 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:43,250 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:43,250 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:43,250 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0a326de6ce4749f797a904aedf1ef780, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/20f2be5d29364c12a94f2e890a16da2f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cf13556f99ef4b77b318b774b0ff8857] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=36.8 K 2024-11-11T23:54:43,250 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a326de6ce4749f797a904aedf1ef780, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731369279584 2024-11-11T23:54:43,251 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20f2be5d29364c12a94f2e890a16da2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731369279712 2024-11-11T23:54:43,251 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf13556f99ef4b77b318b774b0ff8857, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731369280899 2024-11-11T23:54:43,255 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into 14f1b2764317489199814c6b4111433b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:43,255 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:43,255 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=13, startTime=1731369282814; duration=0sec 2024-11-11T23:54:43,255 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:43,255 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:43,258 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:43,259 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/5c83446db4604d7883ef9db8e8bcdd48 is 50, key is test_row_0/C:col10/1731369281536/Put/seqid=0 2024-11-11T23:54:43,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742123_1299 (size=13153) 2024-11-11T23:54:43,353 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/8ceec1eb3ff7442e8118a96385313b95 2024-11-11T23:54:43,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6b2e756c88e0473c92faf424db8960f9 is 50, key is test_row_0/B:col10/1731369281546/Put/seqid=0 2024-11-11T23:54:43,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742124_1300 (size=12301) 2024-11-11T23:54:43,668 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/5c83446db4604d7883ef9db8e8bcdd48 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/5c83446db4604d7883ef9db8e8bcdd48 2024-11-11T23:54:43,672 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 5c83446db4604d7883ef9db8e8bcdd48(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:43,672 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:43,672 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=13, startTime=1731369282814; duration=0sec 2024-11-11T23:54:43,672 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:43,672 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:43,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:43,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369343723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369343724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369343724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369343726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,769 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6b2e756c88e0473c92faf424db8960f9 2024-11-11T23:54:43,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/33cda2b1013940ed99b5f404c602a4a4 is 50, key is test_row_0/C:col10/1731369281546/Put/seqid=0 2024-11-11T23:54:43,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742125_1301 (size=12301) 2024-11-11T23:54:43,779 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/33cda2b1013940ed99b5f404c602a4a4 2024-11-11T23:54:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/8ceec1eb3ff7442e8118a96385313b95 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/8ceec1eb3ff7442e8118a96385313b95 2024-11-11T23:54:43,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/8ceec1eb3ff7442e8118a96385313b95, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T23:54:43,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/6b2e756c88e0473c92faf424db8960f9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6b2e756c88e0473c92faf424db8960f9 2024-11-11T23:54:43,790 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6b2e756c88e0473c92faf424db8960f9, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T23:54:43,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/33cda2b1013940ed99b5f404c602a4a4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/33cda2b1013940ed99b5f404c602a4a4 2024-11-11T23:54:43,794 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/33cda2b1013940ed99b5f404c602a4a4, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T23:54:43,795 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1311b3c706ba19435848d682254b17d6 in 851ms, sequenceid=473, compaction requested=false 2024-11-11T23:54:43,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:43,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:43,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T23:54:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T23:54:43,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T23:54:43,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7690 sec 2024-11-11T23:54:43,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7730 sec 2024-11-11T23:54:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:43,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T23:54:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:43,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:43,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/480ce8b05d4a4a748c0c595e3777daf0 is 50, key is test_row_0/A:col10/1731369283836/Put/seqid=0 2024-11-11T23:54:43,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742126_1302 (size=12301) 2024-11-11T23:54:43,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/480ce8b05d4a4a748c0c595e3777daf0 2024-11-11T23:54:43,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369343853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369343854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369343855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369343857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ab1a2426c9cd4eb69beb89d3daed40a7 is 50, key is test_row_0/B:col10/1731369283836/Put/seqid=0 2024-11-11T23:54:43,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742127_1303 (size=12301) 2024-11-11T23:54:43,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ab1a2426c9cd4eb69beb89d3daed40a7 2024-11-11T23:54:43,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b4e713bc31514103933b53abe6ff1b6c is 50, key is test_row_0/C:col10/1731369283836/Put/seqid=0 2024-11-11T23:54:43,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742128_1304 (size=12301) 2024-11-11T23:54:43,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b4e713bc31514103933b53abe6ff1b6c 2024-11-11T23:54:43,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/480ce8b05d4a4a748c0c595e3777daf0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/480ce8b05d4a4a748c0c595e3777daf0 2024-11-11T23:54:43,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/480ce8b05d4a4a748c0c595e3777daf0, entries=150, sequenceid=492, filesize=12.0 K 2024-11-11T23:54:43,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ab1a2426c9cd4eb69beb89d3daed40a7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ab1a2426c9cd4eb69beb89d3daed40a7 2024-11-11T23:54:43,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ab1a2426c9cd4eb69beb89d3daed40a7, entries=150, sequenceid=492, filesize=12.0 K 2024-11-11T23:54:43,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/b4e713bc31514103933b53abe6ff1b6c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b4e713bc31514103933b53abe6ff1b6c 2024-11-11T23:54:43,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b4e713bc31514103933b53abe6ff1b6c, entries=150, sequenceid=492, filesize=12.0 K 2024-11-11T23:54:43,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1311b3c706ba19435848d682254b17d6 in 68ms, sequenceid=492, compaction requested=true 2024-11-11T23:54:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:43,906 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:43,906 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:43,907 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:43,907 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:43,907 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:43,907 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/d743d484db354ac7b07f41a9bf48999d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/8ceec1eb3ff7442e8118a96385313b95, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/480ce8b05d4a4a748c0c595e3777daf0] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=36.9 K 2024-11-11T23:54:43,907 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:43,908 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:43,908 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:43,908 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/14f1b2764317489199814c6b4111433b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6b2e756c88e0473c92faf424db8960f9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ab1a2426c9cd4eb69beb89d3daed40a7] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=36.9 K 2024-11-11T23:54:43,908 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d743d484db354ac7b07f41a9bf48999d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731369280899 2024-11-11T23:54:43,908 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 14f1b2764317489199814c6b4111433b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731369280899 2024-11-11T23:54:43,908 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ceec1eb3ff7442e8118a96385313b95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731369281546 2024-11-11T23:54:43,909 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b2e756c88e0473c92faf424db8960f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731369281546 2024-11-11T23:54:43,909 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 480ce8b05d4a4a748c0c595e3777daf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731369283721 2024-11-11T23:54:43,909 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ab1a2426c9cd4eb69beb89d3daed40a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731369283721 2024-11-11T23:54:43,915 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:43,915 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/c77e7361c0a14c04a953915fc3bf7d14 is 50, key is test_row_0/B:col10/1731369283836/Put/seqid=0 2024-11-11T23:54:43,917 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:43,917 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/b2d8920f1ee644b99ee331dcd10df28a is 50, key is test_row_0/A:col10/1731369283836/Put/seqid=0 2024-11-11T23:54:43,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742130_1306 (size=13255) 2024-11-11T23:54:43,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742129_1305 (size=13255) 2024-11-11T23:54:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:43,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T23:54:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:43,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/92fd15007df340cfb4c6b2c1a55d7dca is 50, key is test_row_0/A:col10/1731369283946/Put/seqid=0 2024-11-11T23:54:43,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742131_1307 (size=14741) 2024-11-11T23:54:43,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369343974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369343975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369343976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369343977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:43,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369343979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:43,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369344085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369344085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369344085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369344085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369344094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T23:54:44,175 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:44,175 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T23:54:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T23:54:44,177 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:44,177 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:44,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T23:54:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369344295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369344295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369344295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369344295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369344304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,327 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/b2d8920f1ee644b99ee331dcd10df28a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/b2d8920f1ee644b99ee331dcd10df28a 2024-11-11T23:54:44,327 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/c77e7361c0a14c04a953915fc3bf7d14 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c77e7361c0a14c04a953915fc3bf7d14 2024-11-11T23:54:44,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T23:54:44,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:44,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,332 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into c77e7361c0a14c04a953915fc3bf7d14(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:44,332 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into b2d8920f1ee644b99ee331dcd10df28a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:44,332 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:44,332 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:44,332 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=13, startTime=1731369283906; duration=0sec 2024-11-11T23:54:44,332 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=13, startTime=1731369283906; duration=0sec 2024-11-11T23:54:44,333 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:44,333 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:44,333 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:54:44,333 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:44,333 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,334 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:54:44,334 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:44,334 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,334 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/5c83446db4604d7883ef9db8e8bcdd48, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/33cda2b1013940ed99b5f404c602a4a4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b4e713bc31514103933b53abe6ff1b6c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=36.9 K 2024-11-11T23:54:44,334 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c83446db4604d7883ef9db8e8bcdd48, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731369280899 2024-11-11T23:54:44,335 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 33cda2b1013940ed99b5f404c602a4a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731369281546 2024-11-11T23:54:44,335 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting b4e713bc31514103933b53abe6ff1b6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731369283721 2024-11-11T23:54:44,341 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:44,341 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/7c5f63c997db4527b282b478a4ac6c7f is 50, key is test_row_0/C:col10/1731369283836/Put/seqid=0 2024-11-11T23:54:44,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742132_1308 (size=13255) 2024-11-11T23:54:44,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/92fd15007df340cfb4c6b2c1a55d7dca 2024-11-11T23:54:44,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/d1fb417f38514d8b939e3ecd7cc572bd is 50, key is test_row_0/B:col10/1731369283946/Put/seqid=0 2024-11-11T23:54:44,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742133_1309 (size=12301) 2024-11-11T23:54:44,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T23:54:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T23:54:44,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369344605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369344605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369344605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369344605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369344614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:44,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T23:54:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,759 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/7c5f63c997db4527b282b478a4ac6c7f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/7c5f63c997db4527b282b478a4ac6c7f 2024-11-11T23:54:44,763 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 7c5f63c997db4527b282b478a4ac6c7f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:44,763 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:44,763 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=13, startTime=1731369283906; duration=0sec 2024-11-11T23:54:44,763 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:44,763 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:44,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/d1fb417f38514d8b939e3ecd7cc572bd 2024-11-11T23:54:44,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/0fb01f231c284648bafd559a53dc2ea1 is 50, key is test_row_0/C:col10/1731369283946/Put/seqid=0 2024-11-11T23:54:44,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742134_1310 (size=12301) 2024-11-11T23:54:44,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T23:54:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T23:54:44,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T23:54:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:44,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:45,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T23:54:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:45,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:45,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369345114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:45,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:45,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369345114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369345114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:45,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369345114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:45,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369345124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:45,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/0fb01f231c284648bafd559a53dc2ea1 2024-11-11T23:54:45,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/92fd15007df340cfb4c6b2c1a55d7dca as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/92fd15007df340cfb4c6b2c1a55d7dca 2024-11-11T23:54:45,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/92fd15007df340cfb4c6b2c1a55d7dca, entries=200, sequenceid=509, filesize=14.4 K 2024-11-11T23:54:45,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/d1fb417f38514d8b939e3ecd7cc572bd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/d1fb417f38514d8b939e3ecd7cc572bd 2024-11-11T23:54:45,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/d1fb417f38514d8b939e3ecd7cc572bd, entries=150, sequenceid=509, filesize=12.0 K 2024-11-11T23:54:45,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/0fb01f231c284648bafd559a53dc2ea1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0fb01f231c284648bafd559a53dc2ea1 2024-11-11T23:54:45,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0fb01f231c284648bafd559a53dc2ea1, entries=150, sequenceid=509, filesize=12.0 K 2024-11-11T23:54:45,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1311b3c706ba19435848d682254b17d6 in 1256ms, sequenceid=509, compaction requested=false 2024-11-11T23:54:45,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:45,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T23:54:45,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:45,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T23:54:45,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:45,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:45,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:45,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:45,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:45,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:45,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/ce31d19a92e749f889def9ebd7230198 is 50, key is test_row_0/A:col10/1731369283975/Put/seqid=0 2024-11-11T23:54:45,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742135_1311 (size=12301) 2024-11-11T23:54:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T23:54:45,668 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/ce31d19a92e749f889def9ebd7230198 2024-11-11T23:54:45,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ad78374e41384184a42eea8c95e2706c is 50, key is test_row_0/B:col10/1731369283975/Put/seqid=0 2024-11-11T23:54:45,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742136_1312 (size=12301) 2024-11-11T23:54:46,081 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ad78374e41384184a42eea8c95e2706c 2024-11-11T23:54:46,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/e27295f365f9410db828bdd7c9af5c48 is 50, key is test_row_0/C:col10/1731369283975/Put/seqid=0 2024-11-11T23:54:46,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742137_1313 (size=12301) 2024-11-11T23:54:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:46,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369346140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369346140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369346148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369346150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369346150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369346254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369346254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369346265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369346266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369346266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T23:54:46,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369346464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369346465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369346476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369346476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369346476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,505 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/e27295f365f9410db828bdd7c9af5c48 2024-11-11T23:54:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/ce31d19a92e749f889def9ebd7230198 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/ce31d19a92e749f889def9ebd7230198 2024-11-11T23:54:46,536 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/ce31d19a92e749f889def9ebd7230198, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T23:54:46,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/ad78374e41384184a42eea8c95e2706c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ad78374e41384184a42eea8c95e2706c 2024-11-11T23:54:46,550 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ad78374e41384184a42eea8c95e2706c, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T23:54:46,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/e27295f365f9410db828bdd7c9af5c48 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/e27295f365f9410db828bdd7c9af5c48 2024-11-11T23:54:46,573 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/e27295f365f9410db828bdd7c9af5c48, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T23:54:46,574 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1311b3c706ba19435848d682254b17d6 in 1324ms, sequenceid=527, compaction requested=true 2024-11-11T23:54:46,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:46,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:46,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T23:54:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T23:54:46,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T23:54:46,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3990 sec 2024-11-11T23:54:46,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.4030 sec 2024-11-11T23:54:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:46,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T23:54:46,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:46,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:46,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:46,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:46,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:46,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:46,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/3b56fd36f4f744f5a262d9d3386f0be4 is 50, key is test_row_0/A:col10/1731369286777/Put/seqid=0 2024-11-11T23:54:46,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742138_1314 (size=14741) 2024-11-11T23:54:46,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/3b56fd36f4f744f5a262d9d3386f0be4 2024-11-11T23:54:46,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369346805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369346806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369346804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369346807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369346808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/7c3b74f908b74b79a8ba158705da9a65 is 50, key is test_row_0/B:col10/1731369286777/Put/seqid=0 2024-11-11T23:54:46,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742139_1315 (size=12301) 2024-11-11T23:54:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369346924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369346924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369346924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369346924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369346925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369347134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369347134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369347134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369347135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369347135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/7c3b74f908b74b79a8ba158705da9a65 2024-11-11T23:54:47,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/34cbe6ba4e414a71a4d32214243bb4a4 is 50, key is test_row_0/C:col10/1731369286777/Put/seqid=0 2024-11-11T23:54:47,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742140_1316 (size=12301) 2024-11-11T23:54:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369347444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369347444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369347444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369347445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369347445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/34cbe6ba4e414a71a4d32214243bb4a4 2024-11-11T23:54:47,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/3b56fd36f4f744f5a262d9d3386f0be4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3b56fd36f4f744f5a262d9d3386f0be4 2024-11-11T23:54:47,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3b56fd36f4f744f5a262d9d3386f0be4, entries=200, sequenceid=546, filesize=14.4 K 2024-11-11T23:54:47,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/7c3b74f908b74b79a8ba158705da9a65 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7c3b74f908b74b79a8ba158705da9a65 2024-11-11T23:54:47,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7c3b74f908b74b79a8ba158705da9a65, entries=150, sequenceid=546, filesize=12.0 K 2024-11-11T23:54:47,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/34cbe6ba4e414a71a4d32214243bb4a4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/34cbe6ba4e414a71a4d32214243bb4a4 2024-11-11T23:54:47,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/34cbe6ba4e414a71a4d32214243bb4a4, entries=150, sequenceid=546, filesize=12.0 K 2024-11-11T23:54:47,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1311b3c706ba19435848d682254b17d6 in 941ms, sequenceid=546, compaction requested=true 2024-11-11T23:54:47,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:47,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:54:47,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:47,721 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:47,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:54:47,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:47,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1311b3c706ba19435848d682254b17d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:54:47,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T23:54:47,721 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:47,725 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:47,725 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/B is initiating minor compaction (all files) 2024-11-11T23:54:47,725 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/B in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:47,725 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c77e7361c0a14c04a953915fc3bf7d14, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/d1fb417f38514d8b939e3ecd7cc572bd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ad78374e41384184a42eea8c95e2706c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7c3b74f908b74b79a8ba158705da9a65] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=49.0 K 2024-11-11T23:54:47,725 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:47,726 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/A is initiating minor compaction (all files) 2024-11-11T23:54:47,726 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/A in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:47,726 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/b2d8920f1ee644b99ee331dcd10df28a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/92fd15007df340cfb4c6b2c1a55d7dca, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/ce31d19a92e749f889def9ebd7230198, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3b56fd36f4f744f5a262d9d3386f0be4] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=53.7 K 2024-11-11T23:54:47,726 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c77e7361c0a14c04a953915fc3bf7d14, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731369283721 2024-11-11T23:54:47,726 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d8920f1ee644b99ee331dcd10df28a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731369283721 2024-11-11T23:54:47,727 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting d1fb417f38514d8b939e3ecd7cc572bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731369283851 2024-11-11T23:54:47,727 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92fd15007df340cfb4c6b2c1a55d7dca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731369283841 2024-11-11T23:54:47,727 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ad78374e41384184a42eea8c95e2706c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731369283968 2024-11-11T23:54:47,727 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce31d19a92e749f889def9ebd7230198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731369283968 2024-11-11T23:54:47,727 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c3b74f908b74b79a8ba158705da9a65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731369286145 2024-11-11T23:54:47,727 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b56fd36f4f744f5a262d9d3386f0be4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731369286145 2024-11-11T23:54:47,744 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:47,745 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/44b303de82144c1c85deec584fff63c5 is 50, key is test_row_0/A:col10/1731369286777/Put/seqid=0 2024-11-11T23:54:47,752 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#B#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:47,752 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/7f93206d1088405485106e4b8c57fce7 is 50, key is test_row_0/B:col10/1731369286777/Put/seqid=0 2024-11-11T23:54:47,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742141_1317 (size=13391) 2024-11-11T23:54:47,810 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/44b303de82144c1c85deec584fff63c5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/44b303de82144c1c85deec584fff63c5 2024-11-11T23:54:47,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742142_1318 (size=13391) 2024-11-11T23:54:47,817 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/A of 1311b3c706ba19435848d682254b17d6 into 44b303de82144c1c85deec584fff63c5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:47,817 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:47,817 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/A, priority=12, startTime=1731369287721; duration=0sec 2024-11-11T23:54:47,818 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:54:47,818 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:A 2024-11-11T23:54:47,818 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:54:47,821 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:54:47,821 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 1311b3c706ba19435848d682254b17d6/C is initiating minor compaction (all files) 2024-11-11T23:54:47,821 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1311b3c706ba19435848d682254b17d6/C in TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:47,821 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/7c5f63c997db4527b282b478a4ac6c7f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0fb01f231c284648bafd559a53dc2ea1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/e27295f365f9410db828bdd7c9af5c48, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/34cbe6ba4e414a71a4d32214243bb4a4] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp, totalSize=49.0 K 2024-11-11T23:54:47,822 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c5f63c997db4527b282b478a4ac6c7f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731369283721 2024-11-11T23:54:47,822 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fb01f231c284648bafd559a53dc2ea1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731369283851 2024-11-11T23:54:47,822 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/7f93206d1088405485106e4b8c57fce7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7f93206d1088405485106e4b8c57fce7 2024-11-11T23:54:47,822 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e27295f365f9410db828bdd7c9af5c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731369283968 2024-11-11T23:54:47,823 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34cbe6ba4e414a71a4d32214243bb4a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731369286145 2024-11-11T23:54:47,828 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/B of 1311b3c706ba19435848d682254b17d6 into 7f93206d1088405485106e4b8c57fce7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:47,828 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:47,828 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/B, priority=12, startTime=1731369287721; duration=0sec 2024-11-11T23:54:47,828 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:47,828 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:B 2024-11-11T23:54:47,842 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1311b3c706ba19435848d682254b17d6#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:54:47,843 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/31e6f1e099ec4a8a904e078a6d9ca807 is 50, key is test_row_0/C:col10/1731369286777/Put/seqid=0 2024-11-11T23:54:47,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742143_1319 (size=13391) 2024-11-11T23:54:47,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T23:54:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:47,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/85b4926a5f324d2c91334bc324725ccd is 50, key is test_row_0/A:col10/1731369287959/Put/seqid=0 2024-11-11T23:54:47,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742144_1320 (size=14741) 2024-11-11T23:54:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369347981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369347986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369347987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369347992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369347994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369348094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369348105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369348105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369348105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369348105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,281 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/31e6f1e099ec4a8a904e078a6d9ca807 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/31e6f1e099ec4a8a904e078a6d9ca807 2024-11-11T23:54:48,292 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1311b3c706ba19435848d682254b17d6/C of 1311b3c706ba19435848d682254b17d6 into 31e6f1e099ec4a8a904e078a6d9ca807(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:54:48,292 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:48,292 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., storeName=1311b3c706ba19435848d682254b17d6/C, priority=12, startTime=1731369287721; duration=0sec 2024-11-11T23:54:48,292 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:54:48,292 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1311b3c706ba19435848d682254b17d6:C 2024-11-11T23:54:48,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369348305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369348315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369348315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369348315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369348315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T23:54:48,345 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:48,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T23:54:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T23:54:48,348 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:48,348 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:48,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:48,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/85b4926a5f324d2c91334bc324725ccd 2024-11-11T23:54:48,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/288f48e922124a4f88899761736d3400 is 50, key is test_row_0/B:col10/1731369287959/Put/seqid=0 2024-11-11T23:54:48,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742145_1321 (size=12301) 2024-11-11T23:54:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T23:54:48,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T23:54:48,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:48,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:48,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:48,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,508 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-11T23:54:48,508 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:48,508 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,508 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,508 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:48,510 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-11T23:54:48,510 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:48,510 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,510 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:48,510 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-11-11T23:54:48,510 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:48,510 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,510 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:48,512 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-11T23:54:48,512 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:48,512 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,512 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,512 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:48,514 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-11-11T23:54:48,514 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:48,514 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,515 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:48,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:48,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369348614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369348624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369348624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369348624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369348624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:48,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T23:54:48,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:48,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:48,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:48,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T23:54:48,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T23:54:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:48,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/288f48e922124a4f88899761736d3400 2024-11-11T23:54:48,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a434d80724c5485db8814a7c993997ef is 50, key is test_row_0/C:col10/1731369287959/Put/seqid=0 2024-11-11T23:54:48,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742146_1322 (size=12301) 2024-11-11T23:54:48,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T23:54:48,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:48,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:48,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:48,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T23:54:49,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T23:54:49,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:49,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:49,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:49,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:49,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:49,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43872 deadline: 1731369349124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:49,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:49,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:49,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731369349134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43888 deadline: 1731369349134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43900 deadline: 1731369349134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:49,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43810 deadline: 1731369349134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:54:49,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a434d80724c5485db8814a7c993997ef 2024-11-11T23:54:49,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/85b4926a5f324d2c91334bc324725ccd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/85b4926a5f324d2c91334bc324725ccd 2024-11-11T23:54:49,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/85b4926a5f324d2c91334bc324725ccd, entries=200, sequenceid=565, filesize=14.4 K 2024-11-11T23:54:49,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/288f48e922124a4f88899761736d3400 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/288f48e922124a4f88899761736d3400 2024-11-11T23:54:49,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/288f48e922124a4f88899761736d3400, entries=150, sequenceid=565, filesize=12.0 K 2024-11-11T23:54:49,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/a434d80724c5485db8814a7c993997ef as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a434d80724c5485db8814a7c993997ef 2024-11-11T23:54:49,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a434d80724c5485db8814a7c993997ef, entries=150, sequenceid=565, filesize=12.0 K 2024-11-11T23:54:49,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 1311b3c706ba19435848d682254b17d6 in 1287ms, sequenceid=565, compaction requested=false 2024-11-11T23:54:49,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:49,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T23:54:49,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:49,269 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T23:54:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:49,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/43c7f07cb7084156a15d1b55b975313c is 50, key is test_row_0/A:col10/1731369287978/Put/seqid=0 2024-11-11T23:54:49,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742147_1323 (size=12301) 2024-11-11T23:54:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T23:54:49,677 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/43c7f07cb7084156a15d1b55b975313c 2024-11-11T23:54:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/510ef1d9fdd04f669609497d36670205 is 50, key is test_row_0/B:col10/1731369287978/Put/seqid=0 2024-11-11T23:54:49,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742148_1324 (size=12301) 2024-11-11T23:54:49,731 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:54:50,087 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/510ef1d9fdd04f669609497d36670205 2024-11-11T23:54:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/f0291c4335ed43e2a64780cc5f03ddf6 is 50, key is test_row_0/C:col10/1731369287978/Put/seqid=0 2024-11-11T23:54:50,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742149_1325 (size=12301) 2024-11-11T23:54:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:50,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. as already flushing 2024-11-11T23:54:50,137 INFO [Thread-1047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1047. 2024-11-11T23:54:50,137 DEBUG [Thread-1047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:50,137 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,137 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:50,146 INFO [Thread-1039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1039. 2024-11-11T23:54:50,146 DEBUG [Thread-1039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:50,146 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,146 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-11-11T23:54:50,146 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-11T23:54:50,146 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:50,146 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:50,146 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,146 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,146 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:50,146 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:50,147 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:50,152 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-11T23:54:50,152 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:54:50,152 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,152 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:50,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T23:54:50,500 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/f0291c4335ed43e2a64780cc5f03ddf6 2024-11-11T23:54:50,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/43c7f07cb7084156a15d1b55b975313c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/43c7f07cb7084156a15d1b55b975313c 2024-11-11T23:54:50,509 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/43c7f07cb7084156a15d1b55b975313c, entries=150, sequenceid=582, filesize=12.0 K 2024-11-11T23:54:50,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/510ef1d9fdd04f669609497d36670205 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/510ef1d9fdd04f669609497d36670205 2024-11-11T23:54:50,513 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/510ef1d9fdd04f669609497d36670205, entries=150, sequenceid=582, filesize=12.0 K 2024-11-11T23:54:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/f0291c4335ed43e2a64780cc5f03ddf6 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/f0291c4335ed43e2a64780cc5f03ddf6 2024-11-11T23:54:50,516 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/f0291c4335ed43e2a64780cc5f03ddf6, entries=150, sequenceid=582, filesize=12.0 K 2024-11-11T23:54:50,517 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=33.54 KB/34350 for 1311b3c706ba19435848d682254b17d6 in 1248ms, sequenceid=582, compaction requested=true 2024-11-11T23:54:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 1311b3c706ba19435848d682254b17d6: 2024-11-11T23:54:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T23:54:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T23:54:50,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T23:54:50,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1710 sec 2024-11-11T23:54:50,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1750 sec 2024-11-11T23:54:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T23:54:52,516 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3044 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3033 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2949 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3033 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3021 2024-11-11T23:54:52,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T23:54:52,518 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:54:52,518 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:54:52,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:52,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:52,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:54:52,521 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T23:54:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T23:54:52,526 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369292526"}]},"ts":"1731369292526"} 2024-11-11T23:54:52,528 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T23:54:52,528 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T23:54:52,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:54:52,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, UNASSIGN}] 2024-11-11T23:54:52,531 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, UNASSIGN 2024-11-11T23:54:52,532 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=1311b3c706ba19435848d682254b17d6, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:52,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, UNASSIGN because future has completed 2024-11-11T23:54:52,533 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:54:52,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:54:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T23:54:52,688 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:52,688 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:54:52,688 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 1311b3c706ba19435848d682254b17d6, disabling compactions & flushes 2024-11-11T23:54:52,688 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:52,688 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:52,688 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. after waiting 0 ms 2024-11-11T23:54:52,689 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:52,689 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 1311b3c706ba19435848d682254b17d6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T23:54:52,689 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=A 2024-11-11T23:54:52,690 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:52,690 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=B 2024-11-11T23:54:52,690 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:52,690 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1311b3c706ba19435848d682254b17d6, store=C 2024-11-11T23:54:52,690 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:52,696 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/23709c51badc41a1a4527fcb1018beb1 is 50, key is test_row_0/A:col10/1731369290145/Put/seqid=0 2024-11-11T23:54:52,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742150_1326 (size=12301) 2024-11-11T23:54:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T23:54:53,101 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/23709c51badc41a1a4527fcb1018beb1 2024-11-11T23:54:53,114 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/66d148f82f9e46d5bcee4f3e52fc5a2b is 50, key is test_row_0/B:col10/1731369290145/Put/seqid=0 2024-11-11T23:54:53,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742151_1327 (size=12301) 2024-11-11T23:54:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T23:54:53,519 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/66d148f82f9e46d5bcee4f3e52fc5a2b 2024-11-11T23:54:53,532 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/4cfa43b8c65b4637bc8974186f4cb04d is 50, key is test_row_0/C:col10/1731369290145/Put/seqid=0 2024-11-11T23:54:53,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742152_1328 (size=12301) 2024-11-11T23:54:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T23:54:53,936 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/4cfa43b8c65b4637bc8974186f4cb04d 2024-11-11T23:54:53,940 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/A/23709c51badc41a1a4527fcb1018beb1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/23709c51badc41a1a4527fcb1018beb1 2024-11-11T23:54:53,943 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/23709c51badc41a1a4527fcb1018beb1, entries=150, sequenceid=590, filesize=12.0 K 2024-11-11T23:54:53,944 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/B/66d148f82f9e46d5bcee4f3e52fc5a2b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/66d148f82f9e46d5bcee4f3e52fc5a2b 2024-11-11T23:54:53,947 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/66d148f82f9e46d5bcee4f3e52fc5a2b, entries=150, sequenceid=590, filesize=12.0 K 2024-11-11T23:54:53,948 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/.tmp/C/4cfa43b8c65b4637bc8974186f4cb04d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/4cfa43b8c65b4637bc8974186f4cb04d 2024-11-11T23:54:53,951 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/4cfa43b8c65b4637bc8974186f4cb04d, entries=150, sequenceid=590, filesize=12.0 K 2024-11-11T23:54:53,952 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1311b3c706ba19435848d682254b17d6 in 1263ms, sequenceid=590, compaction requested=true 2024-11-11T23:54:53,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e6f8a88727f94855b371dd2771509ed1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/a8505b490e95498da748f5121064d386, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/4af9c2d5bd004442a74e4db3ac1f1e05, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/21da2d9baf1644e5afc29a8db03e4a8f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/dda69eb0e8f147eeb04688e5d97fabc4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/686a8482b74e488aa4e90dc375e96536, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/52b005d9ff7e42628724a54b01b9c933, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/adaefb2eab104bad9a798bbf20df4830, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/994c070fd5c1428880d753aa9f14317f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/fc385286f7474265a65d6a59a8285888, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/74ada8e0b5f9475cb1d0516a19ff9165, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/c7f33abb4fcf4d8f9cecf8be654347a9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e0429883f5674ce4ae54800bb217ad5a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/0a8af059ecd343f49ef1cfc207a597b8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1f5a2195bd1742fbb314035de46fe07d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3c1cb762c95c4a1ca5deb2ddc9ccafdd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/06315ee34c734429a043da7afc11be5d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/77281ab4824b4ea1887ffaa39a4c1236, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/91f5cddf535a49eabd281b8f4f8af792, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/caa90875b79643079efb5e6627c91362, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1cf87a858c06494db516a1e627ef7289, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/7c61117ce7a645d3ae349f0e2909e9fe, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3ed50d43cdee4cecb832011787a27945, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/28ddb97b5ea345cf8909383f4d5a34a8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/d743d484db354ac7b07f41a9bf48999d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1dceb6ef83c84782a16f219b909ce4f5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/8ceec1eb3ff7442e8118a96385313b95, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/b2d8920f1ee644b99ee331dcd10df28a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/480ce8b05d4a4a748c0c595e3777daf0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/92fd15007df340cfb4c6b2c1a55d7dca, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/ce31d19a92e749f889def9ebd7230198, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3b56fd36f4f744f5a262d9d3386f0be4] to archive 2024-11-11T23:54:53,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:54:53,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e6f8a88727f94855b371dd2771509ed1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e6f8a88727f94855b371dd2771509ed1 2024-11-11T23:54:53,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/a8505b490e95498da748f5121064d386 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/a8505b490e95498da748f5121064d386 2024-11-11T23:54:53,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/4af9c2d5bd004442a74e4db3ac1f1e05 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/4af9c2d5bd004442a74e4db3ac1f1e05 2024-11-11T23:54:53,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/21da2d9baf1644e5afc29a8db03e4a8f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/21da2d9baf1644e5afc29a8db03e4a8f 2024-11-11T23:54:53,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/dda69eb0e8f147eeb04688e5d97fabc4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/dda69eb0e8f147eeb04688e5d97fabc4 2024-11-11T23:54:53,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/686a8482b74e488aa4e90dc375e96536 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/686a8482b74e488aa4e90dc375e96536 2024-11-11T23:54:53,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/52b005d9ff7e42628724a54b01b9c933 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/52b005d9ff7e42628724a54b01b9c933 2024-11-11T23:54:53,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/adaefb2eab104bad9a798bbf20df4830 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/adaefb2eab104bad9a798bbf20df4830 2024-11-11T23:54:53,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/994c070fd5c1428880d753aa9f14317f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/994c070fd5c1428880d753aa9f14317f 2024-11-11T23:54:53,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/fc385286f7474265a65d6a59a8285888 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/fc385286f7474265a65d6a59a8285888 2024-11-11T23:54:53,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/74ada8e0b5f9475cb1d0516a19ff9165 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/74ada8e0b5f9475cb1d0516a19ff9165 2024-11-11T23:54:53,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/c7f33abb4fcf4d8f9cecf8be654347a9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/c7f33abb4fcf4d8f9cecf8be654347a9 2024-11-11T23:54:53,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e0429883f5674ce4ae54800bb217ad5a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/e0429883f5674ce4ae54800bb217ad5a 2024-11-11T23:54:53,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/0a8af059ecd343f49ef1cfc207a597b8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/0a8af059ecd343f49ef1cfc207a597b8 2024-11-11T23:54:53,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1f5a2195bd1742fbb314035de46fe07d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1f5a2195bd1742fbb314035de46fe07d 2024-11-11T23:54:53,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3c1cb762c95c4a1ca5deb2ddc9ccafdd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3c1cb762c95c4a1ca5deb2ddc9ccafdd 2024-11-11T23:54:53,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/06315ee34c734429a043da7afc11be5d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/06315ee34c734429a043da7afc11be5d 2024-11-11T23:54:53,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/77281ab4824b4ea1887ffaa39a4c1236 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/77281ab4824b4ea1887ffaa39a4c1236 2024-11-11T23:54:53,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/91f5cddf535a49eabd281b8f4f8af792 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/91f5cddf535a49eabd281b8f4f8af792 2024-11-11T23:54:53,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/caa90875b79643079efb5e6627c91362 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/caa90875b79643079efb5e6627c91362 2024-11-11T23:54:53,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1cf87a858c06494db516a1e627ef7289 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1cf87a858c06494db516a1e627ef7289 2024-11-11T23:54:53,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/7c61117ce7a645d3ae349f0e2909e9fe to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/7c61117ce7a645d3ae349f0e2909e9fe 2024-11-11T23:54:53,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3ed50d43cdee4cecb832011787a27945 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3ed50d43cdee4cecb832011787a27945 2024-11-11T23:54:53,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/28ddb97b5ea345cf8909383f4d5a34a8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/28ddb97b5ea345cf8909383f4d5a34a8 2024-11-11T23:54:53,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/d743d484db354ac7b07f41a9bf48999d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/d743d484db354ac7b07f41a9bf48999d 2024-11-11T23:54:53,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1dceb6ef83c84782a16f219b909ce4f5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/1dceb6ef83c84782a16f219b909ce4f5 2024-11-11T23:54:53,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/8ceec1eb3ff7442e8118a96385313b95 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/8ceec1eb3ff7442e8118a96385313b95 2024-11-11T23:54:53,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/b2d8920f1ee644b99ee331dcd10df28a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/b2d8920f1ee644b99ee331dcd10df28a 2024-11-11T23:54:53,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/480ce8b05d4a4a748c0c595e3777daf0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/480ce8b05d4a4a748c0c595e3777daf0 2024-11-11T23:54:53,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/92fd15007df340cfb4c6b2c1a55d7dca to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/92fd15007df340cfb4c6b2c1a55d7dca 2024-11-11T23:54:53,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/ce31d19a92e749f889def9ebd7230198 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/ce31d19a92e749f889def9ebd7230198 2024-11-11T23:54:53,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3b56fd36f4f744f5a262d9d3386f0be4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/3b56fd36f4f744f5a262d9d3386f0be4 2024-11-11T23:54:53,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/b7ede7d41f5c49dda0ffba66b7ea802d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/fc61fcfb853c46fb9e88797600650fe7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/3101c125ad0a403f9ce519acea1d011c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4583c03d70124bb2b1432e5e2323c5f7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/161f2d23e77b43f181dd85eeb99d14ba, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c0cd556df4c2499b82471d6399682986, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2d9f900350d1413193f0b0c2fe6598d5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2ecdcfd3c8864c98b33b3c1dabed23ca, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/11c7102677b34a98920e099b7fdfee12, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2f09dd50c7ba4d688833ced7ba183001, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/890d13564bf748f380e3c38940b2b2f4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/de110b555d0f41d18251406fd7e62a39, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/356fef595d3a4b11855efb984f4ea61a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8006f53d753344399f6cb6749bae1eaf, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8870a3599da54758ad5763d6aeeb6761, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/9ad21d702612434ea93d0dc922478364, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/25d4131d16f54e219641d97261e4bc5c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/687175a5a247440b8343f66655106064, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/aabefd1b09cd49b8ab9c7197f3549025, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6856f3d81ac940a0980bd4a213d62e57, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4d7397cfec8843a790417172981ce80a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/bf5196c20e7e4ff083d9e417b9a9ada1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6ee0f6f23908430fb5fbce6438ba2317, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/0a0fd247176042268362124ee0823e16, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/14f1b2764317489199814c6b4111433b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ccba75b7808f4429882382c3bedb07e0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6b2e756c88e0473c92faf424db8960f9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c77e7361c0a14c04a953915fc3bf7d14, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ab1a2426c9cd4eb69beb89d3daed40a7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/d1fb417f38514d8b939e3ecd7cc572bd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ad78374e41384184a42eea8c95e2706c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7c3b74f908b74b79a8ba158705da9a65] to archive 2024-11-11T23:54:53,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:54:53,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/b7ede7d41f5c49dda0ffba66b7ea802d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/b7ede7d41f5c49dda0ffba66b7ea802d 2024-11-11T23:54:53,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/fc61fcfb853c46fb9e88797600650fe7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/fc61fcfb853c46fb9e88797600650fe7 2024-11-11T23:54:53,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/3101c125ad0a403f9ce519acea1d011c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/3101c125ad0a403f9ce519acea1d011c 2024-11-11T23:54:53,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4583c03d70124bb2b1432e5e2323c5f7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4583c03d70124bb2b1432e5e2323c5f7 2024-11-11T23:54:53,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/161f2d23e77b43f181dd85eeb99d14ba to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/161f2d23e77b43f181dd85eeb99d14ba 2024-11-11T23:54:53,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c0cd556df4c2499b82471d6399682986 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c0cd556df4c2499b82471d6399682986 2024-11-11T23:54:53,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2d9f900350d1413193f0b0c2fe6598d5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2d9f900350d1413193f0b0c2fe6598d5 2024-11-11T23:54:53,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2ecdcfd3c8864c98b33b3c1dabed23ca to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2ecdcfd3c8864c98b33b3c1dabed23ca 2024-11-11T23:54:53,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/11c7102677b34a98920e099b7fdfee12 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/11c7102677b34a98920e099b7fdfee12 2024-11-11T23:54:53,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2f09dd50c7ba4d688833ced7ba183001 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/2f09dd50c7ba4d688833ced7ba183001 2024-11-11T23:54:53,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/890d13564bf748f380e3c38940b2b2f4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/890d13564bf748f380e3c38940b2b2f4 2024-11-11T23:54:54,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/de110b555d0f41d18251406fd7e62a39 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/de110b555d0f41d18251406fd7e62a39 2024-11-11T23:54:54,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/356fef595d3a4b11855efb984f4ea61a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/356fef595d3a4b11855efb984f4ea61a 2024-11-11T23:54:54,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8006f53d753344399f6cb6749bae1eaf to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8006f53d753344399f6cb6749bae1eaf 2024-11-11T23:54:54,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8870a3599da54758ad5763d6aeeb6761 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/8870a3599da54758ad5763d6aeeb6761 2024-11-11T23:54:54,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/9ad21d702612434ea93d0dc922478364 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/9ad21d702612434ea93d0dc922478364 2024-11-11T23:54:54,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/25d4131d16f54e219641d97261e4bc5c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/25d4131d16f54e219641d97261e4bc5c 2024-11-11T23:54:54,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/687175a5a247440b8343f66655106064 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/687175a5a247440b8343f66655106064 2024-11-11T23:54:54,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/aabefd1b09cd49b8ab9c7197f3549025 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/aabefd1b09cd49b8ab9c7197f3549025 2024-11-11T23:54:54,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6856f3d81ac940a0980bd4a213d62e57 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6856f3d81ac940a0980bd4a213d62e57 2024-11-11T23:54:54,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4d7397cfec8843a790417172981ce80a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/4d7397cfec8843a790417172981ce80a 2024-11-11T23:54:54,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/bf5196c20e7e4ff083d9e417b9a9ada1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/bf5196c20e7e4ff083d9e417b9a9ada1 2024-11-11T23:54:54,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6ee0f6f23908430fb5fbce6438ba2317 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6ee0f6f23908430fb5fbce6438ba2317 2024-11-11T23:54:54,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/0a0fd247176042268362124ee0823e16 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/0a0fd247176042268362124ee0823e16 2024-11-11T23:54:54,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/14f1b2764317489199814c6b4111433b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/14f1b2764317489199814c6b4111433b 2024-11-11T23:54:54,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ccba75b7808f4429882382c3bedb07e0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ccba75b7808f4429882382c3bedb07e0 2024-11-11T23:54:54,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6b2e756c88e0473c92faf424db8960f9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/6b2e756c88e0473c92faf424db8960f9 2024-11-11T23:54:54,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c77e7361c0a14c04a953915fc3bf7d14 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/c77e7361c0a14c04a953915fc3bf7d14 2024-11-11T23:54:54,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ab1a2426c9cd4eb69beb89d3daed40a7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ab1a2426c9cd4eb69beb89d3daed40a7 2024-11-11T23:54:54,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/d1fb417f38514d8b939e3ecd7cc572bd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/d1fb417f38514d8b939e3ecd7cc572bd 2024-11-11T23:54:54,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ad78374e41384184a42eea8c95e2706c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/ad78374e41384184a42eea8c95e2706c 2024-11-11T23:54:54,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7c3b74f908b74b79a8ba158705da9a65 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7c3b74f908b74b79a8ba158705da9a65 2024-11-11T23:54:54,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a610ae9e0967482487174fb77056bb29, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a15f73fc8ba84d0d9fe496ba569b90da, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/9fdfd2a4d08e4155861b600c24ef566f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/25534b251c35404d82d80477926c1f20, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b143b4dce8194423b3646ad485478b45, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/91a7cb4a02404ca0aeb057f458fe8679, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/506396762cc44f90a215db2af73f9578, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/991561d21c5c4db6b794ddf58b70a465, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c93d53d214d445a4bba038bd4b8c852a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/15802501efb4423486e996c6260b9c43, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/64c1fe1b33cd4a0682ae6f201dd1a576, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/93fcd8dbef5d47668f4184d0e2132900, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cc05ae80be494ed9b6ce2e7e69d20994, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c59f42507722481f9b597b2852dcbaa0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/40d7c0a703cc451184fbc5f733baacea, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c6154ed264834268a596ca27750b7e8f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/8edd7a3ad3f6431fbbabf211547f91ad, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2cbe91b04d7f40528b999da2bc14b667, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/04b378d6961a4a4fb549b333d3f619ca, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/94902816f29c4f7988ddaf3a429ff347, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b778f3f24aea4445938e30fbd79ea7ab, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0a326de6ce4749f797a904aedf1ef780, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2981b6a6432f4bd38636d0e243dfea33, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/20f2be5d29364c12a94f2e890a16da2f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/5c83446db4604d7883ef9db8e8bcdd48, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cf13556f99ef4b77b318b774b0ff8857, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/33cda2b1013940ed99b5f404c602a4a4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/7c5f63c997db4527b282b478a4ac6c7f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b4e713bc31514103933b53abe6ff1b6c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0fb01f231c284648bafd559a53dc2ea1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/e27295f365f9410db828bdd7c9af5c48, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/34cbe6ba4e414a71a4d32214243bb4a4] to archive 2024-11-11T23:54:54,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:54:54,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a610ae9e0967482487174fb77056bb29 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a610ae9e0967482487174fb77056bb29 2024-11-11T23:54:54,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a15f73fc8ba84d0d9fe496ba569b90da to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a15f73fc8ba84d0d9fe496ba569b90da 2024-11-11T23:54:54,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/9fdfd2a4d08e4155861b600c24ef566f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/9fdfd2a4d08e4155861b600c24ef566f 2024-11-11T23:54:54,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/25534b251c35404d82d80477926c1f20 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/25534b251c35404d82d80477926c1f20 2024-11-11T23:54:54,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b143b4dce8194423b3646ad485478b45 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b143b4dce8194423b3646ad485478b45 2024-11-11T23:54:54,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/91a7cb4a02404ca0aeb057f458fe8679 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/91a7cb4a02404ca0aeb057f458fe8679 2024-11-11T23:54:54,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/506396762cc44f90a215db2af73f9578 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/506396762cc44f90a215db2af73f9578 2024-11-11T23:54:54,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/991561d21c5c4db6b794ddf58b70a465 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/991561d21c5c4db6b794ddf58b70a465 2024-11-11T23:54:54,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c93d53d214d445a4bba038bd4b8c852a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c93d53d214d445a4bba038bd4b8c852a 2024-11-11T23:54:54,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/15802501efb4423486e996c6260b9c43 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/15802501efb4423486e996c6260b9c43 2024-11-11T23:54:54,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/64c1fe1b33cd4a0682ae6f201dd1a576 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/64c1fe1b33cd4a0682ae6f201dd1a576 2024-11-11T23:54:54,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/93fcd8dbef5d47668f4184d0e2132900 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/93fcd8dbef5d47668f4184d0e2132900 2024-11-11T23:54:54,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cc05ae80be494ed9b6ce2e7e69d20994 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cc05ae80be494ed9b6ce2e7e69d20994 2024-11-11T23:54:54,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c59f42507722481f9b597b2852dcbaa0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c59f42507722481f9b597b2852dcbaa0 2024-11-11T23:54:54,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/40d7c0a703cc451184fbc5f733baacea to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/40d7c0a703cc451184fbc5f733baacea 2024-11-11T23:54:54,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c6154ed264834268a596ca27750b7e8f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/c6154ed264834268a596ca27750b7e8f 2024-11-11T23:54:54,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/8edd7a3ad3f6431fbbabf211547f91ad to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/8edd7a3ad3f6431fbbabf211547f91ad 2024-11-11T23:54:54,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2cbe91b04d7f40528b999da2bc14b667 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2cbe91b04d7f40528b999da2bc14b667 2024-11-11T23:54:54,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/04b378d6961a4a4fb549b333d3f619ca to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/04b378d6961a4a4fb549b333d3f619ca 2024-11-11T23:54:54,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/94902816f29c4f7988ddaf3a429ff347 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/94902816f29c4f7988ddaf3a429ff347 2024-11-11T23:54:54,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b778f3f24aea4445938e30fbd79ea7ab to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b778f3f24aea4445938e30fbd79ea7ab 2024-11-11T23:54:54,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0a326de6ce4749f797a904aedf1ef780 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0a326de6ce4749f797a904aedf1ef780 2024-11-11T23:54:54,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2981b6a6432f4bd38636d0e243dfea33 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/2981b6a6432f4bd38636d0e243dfea33 2024-11-11T23:54:54,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/20f2be5d29364c12a94f2e890a16da2f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/20f2be5d29364c12a94f2e890a16da2f 2024-11-11T23:54:54,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/5c83446db4604d7883ef9db8e8bcdd48 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/5c83446db4604d7883ef9db8e8bcdd48 2024-11-11T23:54:54,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cf13556f99ef4b77b318b774b0ff8857 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/cf13556f99ef4b77b318b774b0ff8857 2024-11-11T23:54:54,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/33cda2b1013940ed99b5f404c602a4a4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/33cda2b1013940ed99b5f404c602a4a4 2024-11-11T23:54:54,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/7c5f63c997db4527b282b478a4ac6c7f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/7c5f63c997db4527b282b478a4ac6c7f 2024-11-11T23:54:54,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b4e713bc31514103933b53abe6ff1b6c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/b4e713bc31514103933b53abe6ff1b6c 2024-11-11T23:54:54,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0fb01f231c284648bafd559a53dc2ea1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/0fb01f231c284648bafd559a53dc2ea1 2024-11-11T23:54:54,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/e27295f365f9410db828bdd7c9af5c48 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/e27295f365f9410db828bdd7c9af5c48 2024-11-11T23:54:54,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/34cbe6ba4e414a71a4d32214243bb4a4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/34cbe6ba4e414a71a4d32214243bb4a4 2024-11-11T23:54:54,051 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/recovered.edits/593.seqid, newMaxSeqId=593, maxSeqId=1 2024-11-11T23:54:54,052 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. 2024-11-11T23:54:54,052 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 1311b3c706ba19435848d682254b17d6: Waiting for close lock at 1731369292688Running coprocessor pre-close hooks at 1731369292688Disabling compacts and flushes for region at 1731369292688Disabling writes for close at 1731369292688Obtaining lock to block concurrent updates at 1731369292689 (+1 ms)Preparing flush snapshotting stores in 1311b3c706ba19435848d682254b17d6 at 1731369292689Finished memstore snapshotting TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731369292690 (+1 ms)Flushing stores of TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6. at 1731369292691 (+1 ms)Flushing 1311b3c706ba19435848d682254b17d6/A: creating writer at 1731369292691Flushing 1311b3c706ba19435848d682254b17d6/A: appending metadata at 1731369292695 (+4 ms)Flushing 1311b3c706ba19435848d682254b17d6/A: closing flushed file at 1731369292695Flushing 1311b3c706ba19435848d682254b17d6/B: creating writer at 1731369293109 (+414 ms)Flushing 1311b3c706ba19435848d682254b17d6/B: appending metadata at 1731369293113 (+4 ms)Flushing 1311b3c706ba19435848d682254b17d6/B: closing flushed file at 1731369293113Flushing 1311b3c706ba19435848d682254b17d6/C: creating writer at 1731369293528 (+415 ms)Flushing 1311b3c706ba19435848d682254b17d6/C: appending metadata at 1731369293532 (+4 ms)Flushing 1311b3c706ba19435848d682254b17d6/C: closing flushed file at 1731369293532Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1faab88b: reopening flushed file at 1731369293940 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71d211a3: reopening flushed file at 1731369293943 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17e3b6a8: reopening flushed file at 1731369293947 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1311b3c706ba19435848d682254b17d6 in 1263ms, sequenceid=590, compaction requested=true at 1731369293952 (+5 ms)Writing region close event to WAL at 1731369294048 (+96 ms)Running coprocessor post-close hooks at 1731369294052 (+4 ms)Closed at 1731369294052 2024-11-11T23:54:54,054 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:54,054 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=1311b3c706ba19435848d682254b17d6, regionState=CLOSED 2024-11-11T23:54:54,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:54:54,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T23:54:54,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 1311b3c706ba19435848d682254b17d6, server=b70d29849faf,42393,1731369201430 in 1.5230 sec 2024-11-11T23:54:54,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-11T23:54:54,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1311b3c706ba19435848d682254b17d6, UNASSIGN in 1.5280 sec 2024-11-11T23:54:54,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T23:54:54,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5330 sec 2024-11-11T23:54:54,064 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369294064"}]},"ts":"1731369294064"} 2024-11-11T23:54:54,065 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T23:54:54,065 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T23:54:54,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5450 sec 2024-11-11T23:54:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T23:54:54,675 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:54,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T23:54:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:54,681 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T23:54:54,682 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:54,684 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:54,686 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/recovered.edits] 2024-11-11T23:54:54,689 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/23709c51badc41a1a4527fcb1018beb1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/23709c51badc41a1a4527fcb1018beb1 2024-11-11T23:54:54,690 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/43c7f07cb7084156a15d1b55b975313c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/43c7f07cb7084156a15d1b55b975313c 2024-11-11T23:54:54,691 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/44b303de82144c1c85deec584fff63c5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/44b303de82144c1c85deec584fff63c5 2024-11-11T23:54:54,692 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/85b4926a5f324d2c91334bc324725ccd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/A/85b4926a5f324d2c91334bc324725ccd 2024-11-11T23:54:54,695 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/288f48e922124a4f88899761736d3400 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/288f48e922124a4f88899761736d3400 2024-11-11T23:54:54,696 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/510ef1d9fdd04f669609497d36670205 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/510ef1d9fdd04f669609497d36670205 2024-11-11T23:54:54,697 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/66d148f82f9e46d5bcee4f3e52fc5a2b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/66d148f82f9e46d5bcee4f3e52fc5a2b 2024-11-11T23:54:54,698 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7f93206d1088405485106e4b8c57fce7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/B/7f93206d1088405485106e4b8c57fce7 2024-11-11T23:54:54,700 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/31e6f1e099ec4a8a904e078a6d9ca807 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/31e6f1e099ec4a8a904e078a6d9ca807 2024-11-11T23:54:54,701 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/4cfa43b8c65b4637bc8974186f4cb04d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/4cfa43b8c65b4637bc8974186f4cb04d 2024-11-11T23:54:54,702 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a434d80724c5485db8814a7c993997ef to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/a434d80724c5485db8814a7c993997ef 2024-11-11T23:54:54,703 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/f0291c4335ed43e2a64780cc5f03ddf6 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/C/f0291c4335ed43e2a64780cc5f03ddf6 2024-11-11T23:54:54,704 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/recovered.edits/593.seqid to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6/recovered.edits/593.seqid 2024-11-11T23:54:54,705 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/1311b3c706ba19435848d682254b17d6 2024-11-11T23:54:54,705 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T23:54:54,707 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:54,709 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T23:54:54,711 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T23:54:54,712 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:54,712 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T23:54:54,712 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731369294712"}]},"ts":"9223372036854775807"} 2024-11-11T23:54:54,714 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T23:54:54,714 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1311b3c706ba19435848d682254b17d6, NAME => 'TestAcidGuarantees,,1731369266309.1311b3c706ba19435848d682254b17d6.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T23:54:54,714 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T23:54:54,714 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731369294714"}]},"ts":"9223372036854775807"} 2024-11-11T23:54:54,715 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T23:54:54,716 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:54,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-11T23:54:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T23:54:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T23:54:54,785 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:54,796 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_809876577_22 at /127.0.0.1:58014 [Waiting for operation #377] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1248936657_22 at /127.0.0.1:56788 [Waiting for operation #297] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=474 (was 530), ProcessCount=11 (was 11), AvailableMemoryMB=4640 (was 4209) - AvailableMemoryMB LEAK? - 2024-11-11T23:54:54,807 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=474, ProcessCount=11, AvailableMemoryMB=4640 2024-11-11T23:54:54,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:54:54,809 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:54:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:54,812 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:54:54,812 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:54,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-11T23:54:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T23:54:54,813 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:54:54,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742153_1329 (size=1061) 2024-11-11T23:54:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T23:54:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T23:54:55,226 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 2024-11-11T23:54:55,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742154_1330 (size=53) 2024-11-11T23:54:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T23:54:55,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:54:55,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ed59ed5c1dfde989ef4a8e10d39d3e5e, disabling compactions & flushes 2024-11-11T23:54:55,640 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:55,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:55,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. after waiting 0 ms 2024-11-11T23:54:55,641 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:55,641 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:55,641 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: Waiting for close lock at 1731369295640Disabling compacts and flushes for region at 1731369295640Disabling writes for close at 1731369295640Writing region close event to WAL at 1731369295641 (+1 ms)Closed at 1731369295641 2024-11-11T23:54:55,644 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:54:55,644 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731369295644"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369295644"}]},"ts":"1731369295644"} 2024-11-11T23:54:55,649 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:54:55,650 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:54:55,650 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369295650"}]},"ts":"1731369295650"} 2024-11-11T23:54:55,651 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T23:54:55,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, ASSIGN}] 2024-11-11T23:54:55,653 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, ASSIGN 2024-11-11T23:54:55,653 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, ASSIGN; state=OFFLINE, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=false 2024-11-11T23:54:55,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=ed59ed5c1dfde989ef4a8e10d39d3e5e, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:55,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, ASSIGN because future has completed 2024-11-11T23:54:55,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:54:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T23:54:55,968 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:55,968 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:54:55,969 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,969 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:54:55,969 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,969 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,972 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,974 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:55,974 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed59ed5c1dfde989ef4a8e10d39d3e5e columnFamilyName A 2024-11-11T23:54:55,974 DEBUG [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:55,975 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(327): Store=ed59ed5c1dfde989ef4a8e10d39d3e5e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:55,975 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,977 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:55,977 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed59ed5c1dfde989ef4a8e10d39d3e5e columnFamilyName B 2024-11-11T23:54:55,977 DEBUG [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:55,978 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(327): Store=ed59ed5c1dfde989ef4a8e10d39d3e5e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:55,978 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,979 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:55,979 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed59ed5c1dfde989ef4a8e10d39d3e5e columnFamilyName C 2024-11-11T23:54:55,979 DEBUG [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:55,980 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(327): Store=ed59ed5c1dfde989ef4a8e10d39d3e5e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:55,980 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:55,980 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,981 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,981 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,982 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,982 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,983 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:54:55,984 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,986 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:54:55,986 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened ed59ed5c1dfde989ef4a8e10d39d3e5e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70416261, jitterRate=0.04928405582904816}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:54:55,986 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:55,987 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: Running coprocessor pre-open hook at 1731369295969Writing region info on filesystem at 1731369295969Initializing all the Stores at 1731369295971 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369295971Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369295971Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369295971Cleaning up temporary data from old regions at 1731369295982 (+11 ms)Running coprocessor post-open hooks at 1731369295986 (+4 ms)Region opened successfully at 1731369295987 (+1 ms) 2024-11-11T23:54:55,988 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., pid=85, masterSystemTime=1731369295964 2024-11-11T23:54:55,990 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:55,990 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:55,991 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=ed59ed5c1dfde989ef4a8e10d39d3e5e, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:55,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:54:55,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T23:54:55,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 in 184 msec 2024-11-11T23:54:55,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-11T23:54:55,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, ASSIGN in 343 msec 2024-11-11T23:54:55,998 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:54:55,999 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369295998"}]},"ts":"1731369295998"} 2024-11-11T23:54:56,000 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T23:54:56,001 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:54:56,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-11T23:54:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T23:54:56,955 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:56,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:56,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:56,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:56,959 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca1b608, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:56,962 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5694e49d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:56,964 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:56,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:56,967 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T23:54:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b70d29849faf,37327,1731369200577 2024-11-11T23:54:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12491736 2024-11-11T23:54:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T23:54:56,969 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T23:54:56,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:54:56,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:54:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T23:54:56,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742155_1331 (size=1097) 2024-11-11T23:54:57,390 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T23:54:57,391 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T23:54:57,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:54:57,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, REOPEN/MOVE}] 2024-11-11T23:54:57,399 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, REOPEN/MOVE 2024-11-11T23:54:57,400 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=ed59ed5c1dfde989ef4a8e10d39d3e5e, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:57,401 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37327 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=b70d29849faf,42393,1731369201430, table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T23:54:57,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, REOPEN/MOVE because future has completed 2024-11-11T23:54:57,402 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:54:57,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:54:57,554 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,555 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:54:57,555 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing ed59ed5c1dfde989ef4a8e10d39d3e5e, disabling compactions & flushes 2024-11-11T23:54:57,555 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:57,555 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:57,555 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. after waiting 0 ms 2024-11-11T23:54:57,555 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:57,560 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T23:54:57,561 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:57,561 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: Waiting for close lock at 1731369297555Running coprocessor pre-close hooks at 1731369297555Disabling compacts and flushes for region at 1731369297555Disabling writes for close at 1731369297555Writing region close event to WAL at 1731369297555Running coprocessor post-close hooks at 1731369297561 (+6 ms)Closed at 1731369297561 2024-11-11T23:54:57,561 WARN [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: ed59ed5c1dfde989ef4a8e10d39d3e5e to self. 2024-11-11T23:54:57,563 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,564 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=ed59ed5c1dfde989ef4a8e10d39d3e5e, regionState=CLOSED 2024-11-11T23:54:57,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:54:57,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T23:54:57,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 in 165 msec 2024-11-11T23:54:57,571 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, REOPEN/MOVE; state=CLOSED, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=true 2024-11-11T23:54:57,722 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=ed59ed5c1dfde989ef4a8e10d39d3e5e, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:57,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, REOPEN/MOVE because future has completed 2024-11-11T23:54:57,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:54:57,887 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:57,888 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:54:57,888 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,888 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:54:57,888 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,888 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,891 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,892 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:57,893 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed59ed5c1dfde989ef4a8e10d39d3e5e columnFamilyName A 2024-11-11T23:54:57,894 DEBUG [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:57,895 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(327): Store=ed59ed5c1dfde989ef4a8e10d39d3e5e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:57,895 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,896 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:57,896 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed59ed5c1dfde989ef4a8e10d39d3e5e columnFamilyName B 2024-11-11T23:54:57,897 DEBUG [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:57,897 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(327): Store=ed59ed5c1dfde989ef4a8e10d39d3e5e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:57,897 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,898 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:54:57,898 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed59ed5c1dfde989ef4a8e10d39d3e5e columnFamilyName C 2024-11-11T23:54:57,898 DEBUG [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:57,898 INFO [StoreOpener-ed59ed5c1dfde989ef4a8e10d39d3e5e-1 {}] regionserver.HStore(327): Store=ed59ed5c1dfde989ef4a8e10d39d3e5e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:57,898 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:57,898 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,899 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,900 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,900 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,900 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,901 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:54:57,902 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,903 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened ed59ed5c1dfde989ef4a8e10d39d3e5e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61771034, jitterRate=-0.07953986525535583}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:54:57,903 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,903 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: Running coprocessor pre-open hook at 1731369297889Writing region info on filesystem at 1731369297889Initializing all the Stores at 1731369297890 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369297890Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369297890Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369297891 (+1 ms)Cleaning up temporary data from old regions at 1731369297901 (+10 ms)Running coprocessor post-open hooks at 1731369297903 (+2 ms)Region opened successfully at 1731369297903 2024-11-11T23:54:57,904 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., pid=90, masterSystemTime=1731369297880 2024-11-11T23:54:57,905 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:57,905 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:57,906 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=ed59ed5c1dfde989ef4a8e10d39d3e5e, regionState=OPEN, openSeqNum=5, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:54:57,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:54:57,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-11T23:54:57,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 in 182 msec 2024-11-11T23:54:57,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T23:54:57,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, REOPEN/MOVE in 512 msec 2024-11-11T23:54:57,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-11T23:54:57,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-11T23:54:57,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-11-11T23:54:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T23:54:57,916 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:54:57,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e3192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bf90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,920 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b294827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354a6c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,922 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2331c5b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,924 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467e0a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,925 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee71314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,926 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a06566d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,928 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec4f402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,929 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef3786c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,931 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5975d745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,932 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19098534, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,934 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7844a7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,935 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5568b64c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,937 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688441c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,938 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162fbc42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,940 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359a9ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,942 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aac12da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:54:57,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:54:57,944 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:54:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:54:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:54:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563f3424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:54:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:54:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:54:57,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:54:57,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77883828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:54:57,947 DEBUG [Thread-1519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,947 DEBUG [Thread-1530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,947 DEBUG [Thread-1532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,947 DEBUG [Thread-1521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,947 DEBUG [Thread-1524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:54:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,948 DEBUG [Thread-1528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,949 DEBUG [Thread-1517 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-11T23:54:57,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,950 DEBUG [Thread-1513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T23:54:57,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,951 DEBUG [Thread-1515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,952 DEBUG [Thread-1513 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,952 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:54:57,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,952 DEBUG [Thread-1513 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,952 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:54:57,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:54:57,953 DEBUG [Thread-1526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:54:57,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:54:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:54:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:54:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:54:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:57,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:54:57,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:54:57,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:57,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:54:57,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:57,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:54:57,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:54:57,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117acde49e09104f1daed819cab3520cf2_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369297964/Put/seqid=0 2024-11-11T23:54:57,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742156_1332 (size=14594) 2024-11-11T23:54:58,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369298006, max timestamp=1731369298030Num uniques 859; 2024-11-11T23:54:58,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369298006, max timestamp=1731369298030Num uniques 859; 2024-11-11T23:54:58,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369298006, max timestamp=1731369298030Num uniques 859; 2024-11-11T23:54:58,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:54:58,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:54:58,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:54:58,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T23:54:58,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369358048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369358059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369358059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369358060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369358062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:54:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369358174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369358175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369358175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369358175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369358175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:58,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:54:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T23:54:58,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369358385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369358385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369358385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369358385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369358385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:58,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117acde49e09104f1daed819cab3520cf2_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117acde49e09104f1daed819cab3520cf2_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:58,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/2171e88a947b48a68c9206e8469beda5, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:54:58,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/2171e88a947b48a68c9206e8469beda5 is 175, key is test_row_0/A:col10/1731369297964/Put/seqid=0 2024-11-11T23:54:58,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742157_1333 (size=39549) 2024-11-11T23:54:58,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:54:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:54:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T23:54:58,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369358695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369358695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369358695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369358695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369358704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:58,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:54:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,803 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/2171e88a947b48a68c9206e8469beda5 2024-11-11T23:54:58,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/10bc72e24cb64066b90714e3851eafd5 is 50, key is test_row_0/B:col10/1731369297964/Put/seqid=0 2024-11-11T23:54:58,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742158_1334 (size=12001) 2024-11-11T23:54:58,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:54:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:58,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:59,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:59,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:59,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:54:59,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:59,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:59,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T23:54:59,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:54:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:59,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:59,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369359204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369359204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:59,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369359204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:59,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369359214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:54:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369359215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:54:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:54:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:54:59,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/10bc72e24cb64066b90714e3851eafd5 2024-11-11T23:54:59,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/b91a59dbbe2c4134a8653ce2262723f4 is 50, key is test_row_0/C:col10/1731369297964/Put/seqid=0 2024-11-11T23:54:59,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742159_1335 (size=12001) 2024-11-11T23:54:59,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/b91a59dbbe2c4134a8653ce2262723f4 2024-11-11T23:54:59,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/2171e88a947b48a68c9206e8469beda5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/2171e88a947b48a68c9206e8469beda5 2024-11-11T23:54:59,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/2171e88a947b48a68c9206e8469beda5, entries=200, sequenceid=14, filesize=38.6 K 2024-11-11T23:54:59,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/10bc72e24cb64066b90714e3851eafd5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/10bc72e24cb64066b90714e3851eafd5 2024-11-11T23:54:59,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/10bc72e24cb64066b90714e3851eafd5, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T23:54:59,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/b91a59dbbe2c4134a8653ce2262723f4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/b91a59dbbe2c4134a8653ce2262723f4 2024-11-11T23:54:59,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/b91a59dbbe2c4134a8653ce2262723f4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T23:54:59,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1311ms, sequenceid=14, compaction requested=false 2024-11-11T23:54:59,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:54:59,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T23:54:59,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:54:59,328 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T23:54:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:54:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:54:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:54:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:54:59,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ba96436c65434fc780c57e57bc68588c_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369298046/Put/seqid=0 2024-11-11T23:54:59,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742160_1336 (size=12154) 2024-11-11T23:54:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:59,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ba96436c65434fc780c57e57bc68588c_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ba96436c65434fc780c57e57bc68588c_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:54:59,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/a9ccacf3d92141f3aa843f097519f744, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:54:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/a9ccacf3d92141f3aa843f097519f744 is 175, key is test_row_0/A:col10/1731369298046/Put/seqid=0 2024-11-11T23:54:59,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742161_1337 (size=30955) 2024-11-11T23:54:59,575 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:54:59,748 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/a9ccacf3d92141f3aa843f097519f744 2024-11-11T23:54:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/9dcd1ba5f11f4cc9839b18d564929c51 is 50, key is test_row_0/B:col10/1731369298046/Put/seqid=0 2024-11-11T23:54:59,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742162_1338 (size=12001) 2024-11-11T23:55:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T23:55:00,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/9dcd1ba5f11f4cc9839b18d564929c51 2024-11-11T23:55:00,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/86bb4b69fb0e468186c8d842e433a903 is 50, key is test_row_0/C:col10/1731369298046/Put/seqid=0 2024-11-11T23:55:00,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742163_1339 (size=12001) 2024-11-11T23:55:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:00,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369360234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369360241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,248 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369360247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369360248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369360249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369360354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369360365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369360365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369360365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,572 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/86bb4b69fb0e468186c8d842e433a903 2024-11-11T23:55:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/a9ccacf3d92141f3aa843f097519f744 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/a9ccacf3d92141f3aa843f097519f744 2024-11-11T23:55:00,578 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/a9ccacf3d92141f3aa843f097519f744, entries=150, sequenceid=50, filesize=30.2 K 2024-11-11T23:55:00,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/9dcd1ba5f11f4cc9839b18d564929c51 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/9dcd1ba5f11f4cc9839b18d564929c51 2024-11-11T23:55:00,582 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/9dcd1ba5f11f4cc9839b18d564929c51, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T23:55:00,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/86bb4b69fb0e468186c8d842e433a903 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/86bb4b69fb0e468186c8d842e433a903 2024-11-11T23:55:00,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369360574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/86bb4b69fb0e468186c8d842e433a903, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T23:55:00,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1259ms, sequenceid=50, compaction requested=false 2024-11-11T23:55:00,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:00,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:00,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-11T23:55:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-11T23:55:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:00,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T23:55:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:00,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T23:55:00,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6360 sec 2024-11-11T23:55:00,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.6440 sec 2024-11-11T23:55:00,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146b51a98773a4ee298a580549f061379_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369300246/Put/seqid=0 2024-11-11T23:55:00,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742164_1340 (size=14594) 2024-11-11T23:55:00,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369300628, max timestamp=1731369300646Num uniques 859; 2024-11-11T23:55:00,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369300628, max timestamp=1731369300646Num uniques 859; 2024-11-11T23:55:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369360643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:55:00,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:55:00,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369300628, max timestamp=1731369300646Num uniques 859; 2024-11-11T23:55:00,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:55:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369360707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369360713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369360754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369360824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369360824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369360894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:00,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369360964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:01,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146b51a98773a4ee298a580549f061379_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146b51a98773a4ee298a580549f061379_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:01,014 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/0f6a2c87c993497a9efe4d11c4d742d0, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:01,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/0f6a2c87c993497a9efe4d11c4d742d0 is 175, key is test_row_0/A:col10/1731369300246/Put/seqid=0 2024-11-11T23:55:01,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742165_1341 (size=39549) 2024-11-11T23:55:01,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369361044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369361044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369361274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369361355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369361355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,419 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/0f6a2c87c993497a9efe4d11c4d742d0 2024-11-11T23:55:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369361414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/77ec39faae8140f2a67f788a38c66882 is 50, key is test_row_0/B:col10/1731369300246/Put/seqid=0 2024-11-11T23:55:01,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742166_1342 (size=12001) 2024-11-11T23:55:01,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369361794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/77ec39faae8140f2a67f788a38c66882 2024-11-11T23:55:01,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/fbbf996958034b8c80cef599a96a7640 is 50, key is test_row_0/C:col10/1731369300246/Put/seqid=0 2024-11-11T23:55:01,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742167_1343 (size=12001) 2024-11-11T23:55:01,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369361875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:01,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369361875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:01,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T23:55:02,115 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:02,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-11T23:55:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T23:55:02,118 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:02,118 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:02,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T23:55:02,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/fbbf996958034b8c80cef599a96a7640 2024-11-11T23:55:02,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/0f6a2c87c993497a9efe4d11c4d742d0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/0f6a2c87c993497a9efe4d11c4d742d0 2024-11-11T23:55:02,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/0f6a2c87c993497a9efe4d11c4d742d0, entries=200, sequenceid=61, filesize=38.6 K 2024-11-11T23:55:02,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/77ec39faae8140f2a67f788a38c66882 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/77ec39faae8140f2a67f788a38c66882 2024-11-11T23:55:02,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/77ec39faae8140f2a67f788a38c66882, entries=150, sequenceid=61, filesize=11.7 K 2024-11-11T23:55:02,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/fbbf996958034b8c80cef599a96a7640 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/fbbf996958034b8c80cef599a96a7640 2024-11-11T23:55:02,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/fbbf996958034b8c80cef599a96a7640, entries=150, sequenceid=61, filesize=11.7 K 2024-11-11T23:55:02,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1663ms, sequenceid=61, compaction requested=true 2024-11-11T23:55:02,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:02,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:02,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:02,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:02,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:02,252 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:02,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:02,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T23:55:02,253 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:02,253 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:02,253 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:02,253 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/A is initiating minor compaction (all files) 2024-11-11T23:55:02,253 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/B is initiating minor compaction (all files) 2024-11-11T23:55:02,253 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/A in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,253 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/B in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,253 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/2171e88a947b48a68c9206e8469beda5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/a9ccacf3d92141f3aa843f097519f744, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/0f6a2c87c993497a9efe4d11c4d742d0] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=107.5 K 2024-11-11T23:55:02,253 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/10bc72e24cb64066b90714e3851eafd5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/9dcd1ba5f11f4cc9839b18d564929c51, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/77ec39faae8140f2a67f788a38c66882] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=35.2 K 2024-11-11T23:55:02,253 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,253 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/2171e88a947b48a68c9206e8469beda5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/a9ccacf3d92141f3aa843f097519f744, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/0f6a2c87c993497a9efe4d11c4d742d0] 2024-11-11T23:55:02,254 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2171e88a947b48a68c9206e8469beda5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731369297959 2024-11-11T23:55:02,254 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 10bc72e24cb64066b90714e3851eafd5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731369297959 2024-11-11T23:55:02,254 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9ccacf3d92141f3aa843f097519f744, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731369298046 2024-11-11T23:55:02,254 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcd1ba5f11f4cc9839b18d564929c51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731369298046 2024-11-11T23:55:02,254 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ec39faae8140f2a67f788a38c66882, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731369300245 2024-11-11T23:55:02,254 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6a2c87c993497a9efe4d11c4d742d0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731369300239 2024-11-11T23:55:02,259 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:02,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T23:55:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:02,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:02,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:02,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:02,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:02,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:02,262 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111536e6789aeb149e1bca8206d0e404c11_ed59ed5c1dfde989ef4a8e10d39d3e5e store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:02,264 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111536e6789aeb149e1bca8206d0e404c11_ed59ed5c1dfde989ef4a8e10d39d3e5e, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:02,264 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:02,264 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111536e6789aeb149e1bca8206d0e404c11_ed59ed5c1dfde989ef4a8e10d39d3e5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:02,264 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/388280427d7c49629a2661823c354aff is 50, key is test_row_0/B:col10/1731369300246/Put/seqid=0 2024-11-11T23:55:02,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111192d9a991728646e9ab198fead9900373_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369300695/Put/seqid=0 2024-11-11T23:55:02,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742168_1344 (size=4469) 2024-11-11T23:55:02,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:02,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742169_1345 (size=12104) 2024-11-11T23:55:02,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742170_1346 (size=14594) 2024-11-11T23:55:02,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369362318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:02,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:02,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:02,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T23:55:02,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369362434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369362435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:02,444 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:02,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369362654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:02,673 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#A#compaction#297 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:02,674 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/37a501921ff64ad7a430bb8f80fb65e8 is 175, key is test_row_0/A:col10/1731369300246/Put/seqid=0 2024-11-11T23:55:02,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742171_1347 (size=31058) 2024-11-11T23:55:02,683 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/37a501921ff64ad7a430bb8f80fb65e8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/37a501921ff64ad7a430bb8f80fb65e8 2024-11-11T23:55:02,683 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/388280427d7c49629a2661823c354aff as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/388280427d7c49629a2661823c354aff 2024-11-11T23:55:02,688 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/A of ed59ed5c1dfde989ef4a8e10d39d3e5e into 37a501921ff64ad7a430bb8f80fb65e8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:02,689 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:02,689 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/A, priority=13, startTime=1731369302252; duration=0sec 2024-11-11T23:55:02,689 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:02,689 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:A 2024-11-11T23:55:02,689 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:02,691 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:02,693 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/B of ed59ed5c1dfde989ef4a8e10d39d3e5e into 388280427d7c49629a2661823c354aff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:02,693 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:02,693 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/B, priority=13, startTime=1731369302252; duration=0sec 2024-11-11T23:55:02,693 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:02,693 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:B 2024-11-11T23:55:02,694 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:02,694 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/C is initiating minor compaction (all files) 2024-11-11T23:55:02,694 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/C in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,694 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/b91a59dbbe2c4134a8653ce2262723f4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/86bb4b69fb0e468186c8d842e433a903, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/fbbf996958034b8c80cef599a96a7640] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=35.2 K 2024-11-11T23:55:02,695 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b91a59dbbe2c4134a8653ce2262723f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731369297959 2024-11-11T23:55:02,695 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86bb4b69fb0e468186c8d842e433a903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731369298046 2024-11-11T23:55:02,697 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbbf996958034b8c80cef599a96a7640, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731369300245 2024-11-11T23:55:02,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111192d9a991728646e9ab198fead9900373_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192d9a991728646e9ab198fead9900373_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:02,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6459fd08c74a4a3b8c29cbd9279221f0, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:02,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6459fd08c74a4a3b8c29cbd9279221f0 is 175, key is test_row_0/A:col10/1731369300695/Put/seqid=0 2024-11-11T23:55:02,715 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#C#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:02,716 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/a642ecc595674a91a64fdf1e55ffa5e1 is 50, key is test_row_0/C:col10/1731369300246/Put/seqid=0 2024-11-11T23:55:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742172_1348 (size=39549) 2024-11-11T23:55:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742173_1349 (size=12104) 2024-11-11T23:55:02,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T23:55:02,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369362814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:02,820 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:02,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:02,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:02,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:02,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:02,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369362885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:02,890 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:02,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369362894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:02,901 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:02,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:02,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369362965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:03,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:03,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:03,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,120 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6459fd08c74a4a3b8c29cbd9279221f0 2024-11-11T23:55:03,123 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/a642ecc595674a91a64fdf1e55ffa5e1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a642ecc595674a91a64fdf1e55ffa5e1 2024-11-11T23:55:03,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/ac3c263d92a9406ebaefbee8d910ce92 is 50, key is test_row_0/B:col10/1731369300695/Put/seqid=0 2024-11-11T23:55:03,127 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/C of ed59ed5c1dfde989ef4a8e10d39d3e5e into a642ecc595674a91a64fdf1e55ffa5e1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:03,127 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:03,127 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/C, priority=13, startTime=1731369302252; duration=0sec 2024-11-11T23:55:03,128 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:03,128 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:C 2024-11-11T23:55:03,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742174_1350 (size=12001) 2024-11-11T23:55:03,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:03,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:03,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T23:55:03,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:03,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369363484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:03,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:03,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:03,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/ac3c263d92a9406ebaefbee8d910ce92 2024-11-11T23:55:03,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/e3af6aed99d54f7283f2883b5c9efafd is 50, key is test_row_0/C:col10/1731369300695/Put/seqid=0 2024-11-11T23:55:03,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742175_1351 (size=12001) 2024-11-11T23:55:03,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:03,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:03,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/e3af6aed99d54f7283f2883b5c9efafd 2024-11-11T23:55:03,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:03,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:03,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6459fd08c74a4a3b8c29cbd9279221f0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6459fd08c74a4a3b8c29cbd9279221f0 2024-11-11T23:55:03,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6459fd08c74a4a3b8c29cbd9279221f0, entries=200, sequenceid=96, filesize=38.6 K 2024-11-11T23:55:03,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/ac3c263d92a9406ebaefbee8d910ce92 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ac3c263d92a9406ebaefbee8d910ce92 2024-11-11T23:55:03,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ac3c263d92a9406ebaefbee8d910ce92, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T23:55:03,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/e3af6aed99d54f7283f2883b5c9efafd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/e3af6aed99d54f7283f2883b5c9efafd 2024-11-11T23:55:03,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/e3af6aed99d54f7283f2883b5c9efafd, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T23:55:03,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1705ms, sequenceid=96, compaction requested=false 2024-11-11T23:55:03,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:04,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T23:55:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:04,104 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:55:04,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:04,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:04,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:04,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:04,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:04,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:04,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111759a0ae20df143649df82b9bc0981218_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369302304/Put/seqid=0 2024-11-11T23:55:04,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742176_1352 (size=12154) 2024-11-11T23:55:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T23:55:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:04,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:04,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:04,521 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111759a0ae20df143649df82b9bc0981218_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111759a0ae20df143649df82b9bc0981218_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:04,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/ef956f0de1be41dc8c2be7366781706b, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:04,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/ef956f0de1be41dc8c2be7366781706b is 175, key is test_row_0/A:col10/1731369302304/Put/seqid=0 2024-11-11T23:55:04,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742177_1353 (size=30955) 2024-11-11T23:55:04,534 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/ef956f0de1be41dc8c2be7366781706b 2024-11-11T23:55:04,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0aafc451ecad4f1fa258792b0a986416 is 50, key is test_row_0/B:col10/1731369302304/Put/seqid=0 2024-11-11T23:55:04,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742178_1354 (size=12001) 2024-11-11T23:55:04,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369304519, max timestamp=1731369304548Num uniques 859; 2024-11-11T23:55:04,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:04,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369304519, max timestamp=1731369304548Num uniques 859; 2024-11-11T23:55:04,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369304519, max timestamp=1731369304548Num uniques 859; 2024-11-11T23:55:04,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:04,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:04,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369364616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:04,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369364617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:04,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369364734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:04,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369364744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:04,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369364824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,834 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:04,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:04,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369364894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,905 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:04,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369364915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,921 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:04,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369364944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:04,949 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0aafc451ecad4f1fa258792b0a986416 2024-11-11T23:55:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/1d51b3c7a589423385d6d7a9daf9bf14 is 50, key is test_row_0/C:col10/1731369302304/Put/seqid=0 2024-11-11T23:55:04,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742179_1355 (size=12001) 2024-11-11T23:55:04,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369364964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:05,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369365254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:05,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369365285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:05,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:05,361 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/1d51b3c7a589423385d6d7a9daf9bf14 2024-11-11T23:55:05,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/ef956f0de1be41dc8c2be7366781706b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/ef956f0de1be41dc8c2be7366781706b 2024-11-11T23:55:05,368 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/ef956f0de1be41dc8c2be7366781706b, entries=150, sequenceid=109, filesize=30.2 K 2024-11-11T23:55:05,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0aafc451ecad4f1fa258792b0a986416 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aafc451ecad4f1fa258792b0a986416 2024-11-11T23:55:05,372 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aafc451ecad4f1fa258792b0a986416, entries=150, sequenceid=109, filesize=11.7 K 2024-11-11T23:55:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/1d51b3c7a589423385d6d7a9daf9bf14 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1d51b3c7a589423385d6d7a9daf9bf14 2024-11-11T23:55:05,376 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1d51b3c7a589423385d6d7a9daf9bf14, entries=150, sequenceid=109, filesize=11.7 K 2024-11-11T23:55:05,377 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1272ms, sequenceid=109, compaction requested=true 2024-11-11T23:55:05,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:05,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:05,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-11T23:55:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-11T23:55:05,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-11T23:55:05,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2600 sec 2024-11-11T23:55:05,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.2630 sec 2024-11-11T23:55:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:05,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-11-11T23:55:05,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:05,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:05,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:05,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:05,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:05,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:05,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b8c5022ee7f64b3c9188c4cda27c42ef_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369305770/Put/seqid=0 2024-11-11T23:55:05,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742180_1356 (size=14794) 2024-11-11T23:55:05,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369365803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:05,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369365809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369365915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:05,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369365924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:06,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369366134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369366135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:06,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:06,184 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b8c5022ee7f64b3c9188c4cda27c42ef_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b8c5022ee7f64b3c9188c4cda27c42ef_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:06,185 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/7882b59f00fa40cda96e3fd4a7bf1052, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:06,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/7882b59f00fa40cda96e3fd4a7bf1052 is 175, key is test_row_0/A:col10/1731369305770/Put/seqid=0 2024-11-11T23:55:06,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742181_1357 (size=39749) 2024-11-11T23:55:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T23:55:06,294 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:06,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T23:55:06,296 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T23:55:06,296 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:06,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T23:55:06,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T23:55:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:06,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369366454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:06,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369366454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:06,590 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/7882b59f00fa40cda96e3fd4a7bf1052 2024-11-11T23:55:06,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0949c437b2f44ea7a69ddce1412c6f90 is 50, key is test_row_0/B:col10/1731369305770/Put/seqid=0 2024-11-11T23:55:06,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742182_1358 (size=12151) 2024-11-11T23:55:06,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T23:55:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:06,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0949c437b2f44ea7a69ddce1412c6f90 2024-11-11T23:55:06,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/6925f052fe164291a4d30250a03ee079 is 50, key is test_row_0/C:col10/1731369305770/Put/seqid=0 2024-11-11T23:55:06,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742183_1359 (size=12151) 2024-11-11T23:55:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T23:55:06,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T23:55:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:06,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T23:55:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:06,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T23:55:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369366964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:06,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369366974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:06,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:06,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:07,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/6925f052fe164291a4d30250a03ee079 2024-11-11T23:55:07,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/7882b59f00fa40cda96e3fd4a7bf1052 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/7882b59f00fa40cda96e3fd4a7bf1052 2024-11-11T23:55:07,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/7882b59f00fa40cda96e3fd4a7bf1052, entries=200, sequenceid=146, filesize=38.8 K 2024-11-11T23:55:07,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0949c437b2f44ea7a69ddce1412c6f90 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0949c437b2f44ea7a69ddce1412c6f90 2024-11-11T23:55:07,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0949c437b2f44ea7a69ddce1412c6f90, entries=150, sequenceid=146, filesize=11.9 K 2024-11-11T23:55:07,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/6925f052fe164291a4d30250a03ee079 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6925f052fe164291a4d30250a03ee079 2024-11-11T23:55:07,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6925f052fe164291a4d30250a03ee079, entries=150, sequenceid=146, filesize=11.9 K 2024-11-11T23:55:07,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1257ms, sequenceid=146, compaction requested=true 2024-11-11T23:55:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:07,028 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:07,028 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:07,029 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:07,029 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:07,030 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/A is initiating minor compaction (all files) 2024-11-11T23:55:07,030 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/B is initiating minor compaction (all files) 2024-11-11T23:55:07,030 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/B in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:07,030 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/A in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:07,030 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/37a501921ff64ad7a430bb8f80fb65e8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6459fd08c74a4a3b8c29cbd9279221f0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/ef956f0de1be41dc8c2be7366781706b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/7882b59f00fa40cda96e3fd4a7bf1052] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=138.0 K 2024-11-11T23:55:07,030 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/388280427d7c49629a2661823c354aff, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ac3c263d92a9406ebaefbee8d910ce92, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aafc451ecad4f1fa258792b0a986416, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0949c437b2f44ea7a69ddce1412c6f90] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=47.1 K 2024-11-11T23:55:07,030 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:07,030 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/37a501921ff64ad7a430bb8f80fb65e8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6459fd08c74a4a3b8c29cbd9279221f0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/ef956f0de1be41dc8c2be7366781706b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/7882b59f00fa40cda96e3fd4a7bf1052] 2024-11-11T23:55:07,030 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 388280427d7c49629a2661823c354aff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731369300245 2024-11-11T23:55:07,030 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a501921ff64ad7a430bb8f80fb65e8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731369300245 2024-11-11T23:55:07,031 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6459fd08c74a4a3b8c29cbd9279221f0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731369300695 2024-11-11T23:55:07,031 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3c263d92a9406ebaefbee8d910ce92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731369300695 2024-11-11T23:55:07,031 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef956f0de1be41dc8c2be7366781706b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731369302304 2024-11-11T23:55:07,031 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aafc451ecad4f1fa258792b0a986416, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731369302304 2024-11-11T23:55:07,031 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7882b59f00fa40cda96e3fd4a7bf1052, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731369304599 2024-11-11T23:55:07,031 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0949c437b2f44ea7a69ddce1412c6f90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731369304600 2024-11-11T23:55:07,036 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:07,037 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:07,038 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/ae2124ecaf054e4fbc380e206dac01c5 is 50, key is test_row_0/B:col10/1731369305770/Put/seqid=0 2024-11-11T23:55:07,039 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111158bc2fbe2d342b7900c65326e560e63_ed59ed5c1dfde989ef4a8e10d39d3e5e store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:07,041 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111158bc2fbe2d342b7900c65326e560e63_ed59ed5c1dfde989ef4a8e10d39d3e5e, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:07,041 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111158bc2fbe2d342b7900c65326e560e63_ed59ed5c1dfde989ef4a8e10d39d3e5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:07,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742184_1360 (size=12391) 2024-11-11T23:55:07,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742185_1361 (size=4469) 2024-11-11T23:55:07,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T23:55:07,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:07,059 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T23:55:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:07,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117eac9306888f457298ed3329a82da2dc_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369305808/Put/seqid=0 2024-11-11T23:55:07,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742186_1362 (size=12304) 2024-11-11T23:55:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T23:55:07,447 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:07,447 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/3a023efaab064d48a832a66ed559d086 is 175, key is test_row_0/A:col10/1731369305770/Put/seqid=0 2024-11-11T23:55:07,449 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/ae2124ecaf054e4fbc380e206dac01c5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2124ecaf054e4fbc380e206dac01c5 2024-11-11T23:55:07,454 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/B of ed59ed5c1dfde989ef4a8e10d39d3e5e into ae2124ecaf054e4fbc380e206dac01c5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:07,454 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:07,454 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/B, priority=12, startTime=1731369307028; duration=0sec 2024-11-11T23:55:07,454 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:07,454 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:B 2024-11-11T23:55:07,454 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:07,455 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:07,455 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/C is initiating minor compaction (all files) 2024-11-11T23:55:07,455 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/C in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:07,456 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a642ecc595674a91a64fdf1e55ffa5e1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/e3af6aed99d54f7283f2883b5c9efafd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1d51b3c7a589423385d6d7a9daf9bf14, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6925f052fe164291a4d30250a03ee079] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=47.1 K 2024-11-11T23:55:07,456 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a642ecc595674a91a64fdf1e55ffa5e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731369300245 2024-11-11T23:55:07,456 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting e3af6aed99d54f7283f2883b5c9efafd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731369300695 2024-11-11T23:55:07,457 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d51b3c7a589423385d6d7a9daf9bf14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731369302304 2024-11-11T23:55:07,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742187_1363 (size=31345) 2024-11-11T23:55:07,457 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6925f052fe164291a4d30250a03ee079, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731369304600 2024-11-11T23:55:07,465 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:07,465 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/6c0b3743ceac4c81885576a1bd65d9c8 is 50, key is test_row_0/C:col10/1731369305770/Put/seqid=0 2024-11-11T23:55:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:07,472 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117eac9306888f457298ed3329a82da2dc_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117eac9306888f457298ed3329a82da2dc_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:07,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/8f050526188b4117a1f2351a28593316, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:07,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/8f050526188b4117a1f2351a28593316 is 175, key is test_row_0/A:col10/1731369305808/Put/seqid=0 2024-11-11T23:55:07,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742188_1364 (size=12391) 2024-11-11T23:55:07,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742189_1365 (size=31105) 2024-11-11T23:55:07,861 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/3a023efaab064d48a832a66ed559d086 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/3a023efaab064d48a832a66ed559d086 2024-11-11T23:55:07,865 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/A of ed59ed5c1dfde989ef4a8e10d39d3e5e into 3a023efaab064d48a832a66ed559d086(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:07,865 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:07,865 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/A, priority=12, startTime=1731369307028; duration=0sec 2024-11-11T23:55:07,865 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:07,865 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:A 2024-11-11T23:55:07,883 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/6c0b3743ceac4c81885576a1bd65d9c8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6c0b3743ceac4c81885576a1bd65d9c8 2024-11-11T23:55:07,886 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/8f050526188b4117a1f2351a28593316 2024-11-11T23:55:07,889 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/C of ed59ed5c1dfde989ef4a8e10d39d3e5e into 6c0b3743ceac4c81885576a1bd65d9c8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:07,889 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:07,889 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/C, priority=12, startTime=1731369307028; duration=0sec 2024-11-11T23:55:07,889 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:07,889 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:C 2024-11-11T23:55:07,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/cdbe8e9396134913a2d898a98cad1a89 is 50, key is test_row_0/B:col10/1731369305808/Put/seqid=0 2024-11-11T23:55:07,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742190_1366 (size=12151) 2024-11-11T23:55:07,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:07,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:08,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369308026, max timestamp=1731369308055Num uniques 859; 2024-11-11T23:55:08,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:08,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369308026, max timestamp=1731369308055Num uniques 859; 2024-11-11T23:55:08,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:08,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369308026, max timestamp=1731369308055Num uniques 859; 2024-11-11T23:55:08,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:08,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369368118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369368122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369368235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369368235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,306 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/cdbe8e9396134913a2d898a98cad1a89 2024-11-11T23:55:08,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/a80360bc08114be9ba5eddde515f8254 is 50, key is test_row_0/C:col10/1731369305808/Put/seqid=0 2024-11-11T23:55:08,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742191_1367 (size=12151) 2024-11-11T23:55:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T23:55:08,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369368445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369368445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,716 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/a80360bc08114be9ba5eddde515f8254 2024-11-11T23:55:08,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/8f050526188b4117a1f2351a28593316 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/8f050526188b4117a1f2351a28593316 2024-11-11T23:55:08,723 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/8f050526188b4117a1f2351a28593316, entries=150, sequenceid=155, filesize=30.4 K 2024-11-11T23:55:08,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/cdbe8e9396134913a2d898a98cad1a89 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cdbe8e9396134913a2d898a98cad1a89 2024-11-11T23:55:08,726 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cdbe8e9396134913a2d898a98cad1a89, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T23:55:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/a80360bc08114be9ba5eddde515f8254 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a80360bc08114be9ba5eddde515f8254 2024-11-11T23:55:08,729 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a80360bc08114be9ba5eddde515f8254, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T23:55:08,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1671ms, sequenceid=155, compaction requested=false 2024-11-11T23:55:08,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:08,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:08,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T23:55:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T23:55:08,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T23:55:08,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4350 sec 2024-11-11T23:55:08,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.4370 sec 2024-11-11T23:55:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:08,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T23:55:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:08,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111df322d02adec4bfda47bddbf1ec038d2_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369308103/Put/seqid=0 2024-11-11T23:55:08,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742192_1368 (size=14794) 2024-11-11T23:55:08,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369368795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369368795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1731369368874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,876 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369368905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369368914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39290 deadline: 1731369368934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,943 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:08,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39294 deadline: 1731369368944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:08,955 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in ed59ed5c1dfde989ef4a8e10d39d3e5e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:09,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369369124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:09,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369369124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:09,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:09,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111df322d02adec4bfda47bddbf1ec038d2_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df322d02adec4bfda47bddbf1ec038d2_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:09,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/203b85e45fe040389e3a7593158a869c, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:09,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/203b85e45fe040389e3a7593158a869c is 175, key is test_row_0/A:col10/1731369308103/Put/seqid=0 2024-11-11T23:55:09,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742193_1369 (size=39749) 2024-11-11T23:55:09,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369369444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:09,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369369444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:09,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/203b85e45fe040389e3a7593158a869c 2024-11-11T23:55:09,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/85ee8d80d90c410586bdcc03b63c5de5 is 50, key is test_row_0/B:col10/1731369308103/Put/seqid=0 2024-11-11T23:55:09,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742194_1370 (size=12151) 2024-11-11T23:55:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369369964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369369964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:09,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/85ee8d80d90c410586bdcc03b63c5de5 2024-11-11T23:55:10,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/6a3113ab3b4c4055976843f7b12ff414 is 50, key is test_row_0/C:col10/1731369308103/Put/seqid=0 2024-11-11T23:55:10,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742195_1371 (size=12151) 2024-11-11T23:55:10,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/6a3113ab3b4c4055976843f7b12ff414 2024-11-11T23:55:10,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/203b85e45fe040389e3a7593158a869c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/203b85e45fe040389e3a7593158a869c 2024-11-11T23:55:10,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/203b85e45fe040389e3a7593158a869c, entries=200, sequenceid=196, filesize=38.8 K 2024-11-11T23:55:10,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/85ee8d80d90c410586bdcc03b63c5de5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/85ee8d80d90c410586bdcc03b63c5de5 2024-11-11T23:55:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/85ee8d80d90c410586bdcc03b63c5de5, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T23:55:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/6a3113ab3b4c4055976843f7b12ff414 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6a3113ab3b4c4055976843f7b12ff414 2024-11-11T23:55:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6a3113ab3b4c4055976843f7b12ff414, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T23:55:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1653ms, sequenceid=196, compaction requested=true 2024-11-11T23:55:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:10,424 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:10,424 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,425 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:10,425 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:10,425 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/B is initiating minor compaction (all files) 2024-11-11T23:55:10,425 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/A is initiating minor compaction (all files) 2024-11-11T23:55:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,425 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/A in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:10,425 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/B in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:10,425 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/3a023efaab064d48a832a66ed559d086, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/8f050526188b4117a1f2351a28593316, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/203b85e45fe040389e3a7593158a869c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=99.8 K 2024-11-11T23:55:10,425 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2124ecaf054e4fbc380e206dac01c5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cdbe8e9396134913a2d898a98cad1a89, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/85ee8d80d90c410586bdcc03b63c5de5] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=35.8 K 2024-11-11T23:55:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,425 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:10,425 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/3a023efaab064d48a832a66ed559d086, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/8f050526188b4117a1f2351a28593316, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/203b85e45fe040389e3a7593158a869c] 2024-11-11T23:55:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,426 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ae2124ecaf054e4fbc380e206dac01c5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731369304600 2024-11-11T23:55:10,426 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a023efaab064d48a832a66ed559d086, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731369304600 2024-11-11T23:55:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,426 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting cdbe8e9396134913a2d898a98cad1a89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369305800 2024-11-11T23:55:10,426 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f050526188b4117a1f2351a28593316, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369305800 2024-11-11T23:55:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,426 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ee8d80d90c410586bdcc03b63c5de5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731369308103 2024-11-11T23:55:10,426 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203b85e45fe040389e3a7593158a869c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731369308103 2024-11-11T23:55:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,434 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,435 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/6a03b656b2544c2fb3c79e94e8c12c81 is 50, key is test_row_0/B:col10/1731369308103/Put/seqid=0 2024-11-11T23:55:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,435 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,438 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118ef08b9ccc25446cb474ef80c908ec2b_ed59ed5c1dfde989ef4a8e10d39d3e5e store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,440 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118ef08b9ccc25446cb474ef80c908ec2b_ed59ed5c1dfde989ef4a8e10d39d3e5e, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:10,440 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118ef08b9ccc25446cb474ef80c908ec2b_ed59ed5c1dfde989ef4a8e10d39d3e5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742197_1373 (size=4469) 2024-11-11T23:55:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742196_1372 (size=12493) 2024-11-11T23:55:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T23:55:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,465 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T23:55:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T23:55:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T23:55:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T23:55:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,621 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T23:55:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:10,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:10,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a4c18988bd4e43bc9d303905beb31785_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369308793/Put/seqid=0 2024-11-11T23:55:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742198_1374 (size=9814) 2024-11-11T23:55:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T23:55:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,852 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#A#compaction#319 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,853 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/18ff3a470e6d4db59b04057a6ee0ddbd is 175, key is test_row_0/A:col10/1731369308103/Put/seqid=0 2024-11-11T23:55:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,858 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/6a03b656b2544c2fb3c79e94e8c12c81 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/6a03b656b2544c2fb3c79e94e8c12c81 2024-11-11T23:55:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,862 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/B of ed59ed5c1dfde989ef4a8e10d39d3e5e into 6a03b656b2544c2fb3c79e94e8c12c81(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,862 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,862 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/B, priority=13, startTime=1731369310424; duration=0sec 2024-11-11T23:55:10,862 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:10,862 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:B 2024-11-11T23:55:10,863 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,864 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:10,864 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/C is initiating minor compaction (all files) 2024-11-11T23:55:10,864 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/C in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:10,865 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6c0b3743ceac4c81885576a1bd65d9c8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a80360bc08114be9ba5eddde515f8254, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6a3113ab3b4c4055976843f7b12ff414] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=35.8 K 2024-11-11T23:55:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,865 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c0b3743ceac4c81885576a1bd65d9c8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731369304600 2024-11-11T23:55:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,866 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a80360bc08114be9ba5eddde515f8254, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369305800 2024-11-11T23:55:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,866 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3113ab3b4c4055976843f7b12ff414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731369308103 2024-11-11T23:55:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742199_1375 (size=31447) 2024-11-11T23:55:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,887 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,887 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/f72c3af867384a94b31a48afc789fe9d is 50, key is test_row_0/C:col10/1731369308103/Put/seqid=0 2024-11-11T23:55:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742200_1376 (size=12493) 2024-11-11T23:55:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:11,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,038 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a4c18988bd4e43bc9d303905beb31785_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a4c18988bd4e43bc9d303905beb31785_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/809db127b29a45a09a945bfccc34a556, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/809db127b29a45a09a945bfccc34a556 is 175, key is test_row_0/A:col10/1731369308793/Put/seqid=0 2024-11-11T23:55:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742201_1377 (size=22461) 2024-11-11T23:55:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T23:55:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:11,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369311152, max timestamp=1731369311176Num uniques 859; 2024-11-11T23:55:11,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369311152, max timestamp=1731369311176Num uniques 859; 2024-11-11T23:55:11,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369311152, max timestamp=1731369311176Num uniques 859; 2024-11-11T23:55:11,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:11,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:11,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:11,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369311223, max timestamp=1731369311257Num uniques 1068; 2024-11-11T23:55:11,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369311223, max timestamp=1731369311257Num uniques 1068; 2024-11-11T23:55:11,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731369311223, max timestamp=1731369311257Num uniques 1068; 2024-11-11T23:55:11,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-11T23:55:11,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-11T23:55:11,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-11T23:55:11,280 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/18ff3a470e6d4db59b04057a6ee0ddbd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/18ff3a470e6d4db59b04057a6ee0ddbd 2024-11-11T23:55:11,286 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/A of ed59ed5c1dfde989ef4a8e10d39d3e5e into 18ff3a470e6d4db59b04057a6ee0ddbd(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:11,286 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:11,286 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/A, priority=13, startTime=1731369310424; duration=0sec 2024-11-11T23:55:11,286 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:11,286 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:A 2024-11-11T23:55:11,327 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/f72c3af867384a94b31a48afc789fe9d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/f72c3af867384a94b31a48afc789fe9d 2024-11-11T23:55:11,334 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/C of ed59ed5c1dfde989ef4a8e10d39d3e5e into f72c3af867384a94b31a48afc789fe9d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:11,334 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:11,334 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/C, priority=13, startTime=1731369310424; duration=0sec 2024-11-11T23:55:11,334 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:11,334 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:C 2024-11-11T23:55:11,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369371330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:11,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369371332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:11,453 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/809db127b29a45a09a945bfccc34a556 2024-11-11T23:55:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369371445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:11,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0aa3a643cb4e42aaa91e0f448e8e5d36 is 50, key is test_row_0/B:col10/1731369308793/Put/seqid=0 2024-11-11T23:55:11,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742202_1378 (size=9757) 2024-11-11T23:55:11,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369371454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T23:55:11,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369371664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369371674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:11,863 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0aa3a643cb4e42aaa91e0f448e8e5d36 2024-11-11T23:55:11,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/4b6fb97cdd664a18b6c44b527782f64e is 50, key is test_row_0/C:col10/1731369308793/Put/seqid=0 2024-11-11T23:55:11,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742203_1379 (size=9757) 2024-11-11T23:55:11,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369371974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369371985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:12,272 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/4b6fb97cdd664a18b6c44b527782f64e 2024-11-11T23:55:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/809db127b29a45a09a945bfccc34a556 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/809db127b29a45a09a945bfccc34a556 2024-11-11T23:55:12,278 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/809db127b29a45a09a945bfccc34a556, entries=100, sequenceid=204, filesize=21.9 K 2024-11-11T23:55:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/0aa3a643cb4e42aaa91e0f448e8e5d36 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aa3a643cb4e42aaa91e0f448e8e5d36 2024-11-11T23:55:12,281 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aa3a643cb4e42aaa91e0f448e8e5d36, entries=100, sequenceid=204, filesize=9.5 K 2024-11-11T23:55:12,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/4b6fb97cdd664a18b6c44b527782f64e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/4b6fb97cdd664a18b6c44b527782f64e 2024-11-11T23:55:12,285 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/4b6fb97cdd664a18b6c44b527782f64e, entries=100, sequenceid=204, filesize=9.5 K 2024-11-11T23:55:12,286 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1665ms, sequenceid=204, compaction requested=false 2024-11-11T23:55:12,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:12,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:12,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T23:55:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T23:55:12,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T23:55:12,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-11-11T23:55:12,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8220 sec 2024-11-11T23:55:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:12,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-11-11T23:55:12,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:12,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:12,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:12,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:12,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:12,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:12,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f85c51f467d74af39286b5075ed347c6_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369312502/Put/seqid=0 2024-11-11T23:55:12,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742204_1380 (size=14994) 2024-11-11T23:55:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369372523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:12,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369372524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T23:55:12,615 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:12,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T23:55:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T23:55:12,617 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:12,617 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:12,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369372635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369372635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T23:55:12,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:12,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:12,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369372855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:12,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369372855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:12,914 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:12,917 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f85c51f467d74af39286b5075ed347c6_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f85c51f467d74af39286b5075ed347c6_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:12,918 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/d3b9e6f9f1d84149a15a1d88d0f634cf, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:12,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/d3b9e6f9f1d84149a15a1d88d0f634cf is 175, key is test_row_0/A:col10/1731369312502/Put/seqid=0 2024-11-11T23:55:12,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742205_1381 (size=39949) 2024-11-11T23:55:12,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:12,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T23:55:13,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369373164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:13,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369373164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:13,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T23:55:13,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/d3b9e6f9f1d84149a15a1d88d0f634cf 2024-11-11T23:55:13,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/266765c7a7a6421d987f7b896abdab9a is 50, key is test_row_0/B:col10/1731369312502/Put/seqid=0 2024-11-11T23:55:13,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742206_1382 (size=12301) 2024-11-11T23:55:13,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:13,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:13,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369373674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:13,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:13,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:13,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369373684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:13,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/266765c7a7a6421d987f7b896abdab9a 2024-11-11T23:55:13,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/c31bcdf458964c3886bbea3e889cf84a is 50, key is test_row_0/C:col10/1731369312502/Put/seqid=0 2024-11-11T23:55:13,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742207_1383 (size=12301) 2024-11-11T23:55:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T23:55:13,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:13,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:13,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:13,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:14,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:14,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:14,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:14,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:14,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:14,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:14,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/c31bcdf458964c3886bbea3e889cf84a 2024-11-11T23:55:14,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/d3b9e6f9f1d84149a15a1d88d0f634cf as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d3b9e6f9f1d84149a15a1d88d0f634cf 2024-11-11T23:55:14,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d3b9e6f9f1d84149a15a1d88d0f634cf, entries=200, sequenceid=262, filesize=39.0 K 2024-11-11T23:55:14,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/266765c7a7a6421d987f7b896abdab9a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/266765c7a7a6421d987f7b896abdab9a 2024-11-11T23:55:14,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/266765c7a7a6421d987f7b896abdab9a, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T23:55:14,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/c31bcdf458964c3886bbea3e889cf84a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/c31bcdf458964c3886bbea3e889cf84a 2024-11-11T23:55:14,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/c31bcdf458964c3886bbea3e889cf84a, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T23:55:14,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=26.84 KB/27480 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1659ms, sequenceid=262, compaction requested=true 2024-11-11T23:55:14,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:14,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:14,163 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:14,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:14,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:14,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:14,163 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:14,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:14,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:14,164 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:14,164 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:14,164 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/B is initiating minor compaction (all files) 2024-11-11T23:55:14,164 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/A is initiating minor compaction (all files) 2024-11-11T23:55:14,164 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/B in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:14,164 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/A in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:14,164 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/6a03b656b2544c2fb3c79e94e8c12c81, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aa3a643cb4e42aaa91e0f448e8e5d36, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/266765c7a7a6421d987f7b896abdab9a] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=33.7 K 2024-11-11T23:55:14,164 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/18ff3a470e6d4db59b04057a6ee0ddbd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/809db127b29a45a09a945bfccc34a556, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d3b9e6f9f1d84149a15a1d88d0f634cf] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=91.7 K 2024-11-11T23:55:14,164 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:14,164 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/18ff3a470e6d4db59b04057a6ee0ddbd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/809db127b29a45a09a945bfccc34a556, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d3b9e6f9f1d84149a15a1d88d0f634cf] 2024-11-11T23:55:14,164 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a03b656b2544c2fb3c79e94e8c12c81, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731369308103 2024-11-11T23:55:14,165 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ff3a470e6d4db59b04057a6ee0ddbd, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731369308103 2024-11-11T23:55:14,165 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa3a643cb4e42aaa91e0f448e8e5d36, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731369308793 2024-11-11T23:55:14,165 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 266765c7a7a6421d987f7b896abdab9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731369311321 2024-11-11T23:55:14,165 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 809db127b29a45a09a945bfccc34a556, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731369308793 2024-11-11T23:55:14,166 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b9e6f9f1d84149a15a1d88d0f634cf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731369311321 2024-11-11T23:55:14,172 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:14,172 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/e4e645b8081a4fac9b50025f8a93426c is 50, key is test_row_0/B:col10/1731369312502/Put/seqid=0 2024-11-11T23:55:14,173 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:14,177 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d530feefe058464182a9dad22d06c88b_ed59ed5c1dfde989ef4a8e10d39d3e5e store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:14,179 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d530feefe058464182a9dad22d06c88b_ed59ed5c1dfde989ef4a8e10d39d3e5e, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:14,179 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d530feefe058464182a9dad22d06c88b_ed59ed5c1dfde989ef4a8e10d39d3e5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:14,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742208_1384 (size=12745) 2024-11-11T23:55:14,192 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/e4e645b8081a4fac9b50025f8a93426c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e4e645b8081a4fac9b50025f8a93426c 2024-11-11T23:55:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742209_1385 (size=4469) 2024-11-11T23:55:14,199 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/B of ed59ed5c1dfde989ef4a8e10d39d3e5e into e4e645b8081a4fac9b50025f8a93426c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:14,199 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:14,199 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/B, priority=13, startTime=1731369314163; duration=0sec 2024-11-11T23:55:14,199 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:14,199 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:B 2024-11-11T23:55:14,199 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:14,200 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:14,200 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/C is initiating minor compaction (all files) 2024-11-11T23:55:14,200 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/C in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:14,200 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/f72c3af867384a94b31a48afc789fe9d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/4b6fb97cdd664a18b6c44b527782f64e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/c31bcdf458964c3886bbea3e889cf84a] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=33.7 K 2024-11-11T23:55:14,201 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f72c3af867384a94b31a48afc789fe9d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731369308103 2024-11-11T23:55:14,201 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b6fb97cdd664a18b6c44b527782f64e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731369308793 2024-11-11T23:55:14,201 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c31bcdf458964c3886bbea3e889cf84a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731369311321 2024-11-11T23:55:14,206 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:14,206 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/dfc4e9e0bac44acfa2f2923064193872 is 50, key is test_row_0/C:col10/1731369312502/Put/seqid=0 2024-11-11T23:55:14,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742210_1386 (size=12745) 2024-11-11T23:55:14,222 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/dfc4e9e0bac44acfa2f2923064193872 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/dfc4e9e0bac44acfa2f2923064193872 2024-11-11T23:55:14,228 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/C of ed59ed5c1dfde989ef4a8e10d39d3e5e into dfc4e9e0bac44acfa2f2923064193872(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:14,228 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:14,228 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/C, priority=13, startTime=1731369314163; duration=0sec 2024-11-11T23:55:14,228 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:14,228 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:C 2024-11-11T23:55:14,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T23:55:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:14,297 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T23:55:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:14,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119abeaa945fa94f448f827911f3e733df_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369312523/Put/seqid=0 2024-11-11T23:55:14,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742211_1387 (size=12454) 2024-11-11T23:55:14,597 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:14,598 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/c30fd7d740a64e3391a61ab3aa96b653 is 175, key is test_row_0/A:col10/1731369312502/Put/seqid=0 2024-11-11T23:55:14,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742212_1388 (size=31699) 2024-11-11T23:55:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:14,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:14,727 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119abeaa945fa94f448f827911f3e733df_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119abeaa945fa94f448f827911f3e733df_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6b2d5548f0b141e5b211209c73e80db4, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6b2d5548f0b141e5b211209c73e80db4 is 175, key is test_row_0/A:col10/1731369312523/Put/seqid=0 2024-11-11T23:55:14,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742213_1389 (size=31255) 2024-11-11T23:55:14,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369314739, max timestamp=1731369314761Num uniques 859; 2024-11-11T23:55:14,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369314739, max timestamp=1731369314761Num uniques 859; 2024-11-11T23:55:14,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369314739, max timestamp=1731369314761Num uniques 859; 2024-11-11T23:55:14,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:14,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:14,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T23:55:14,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369314806, max timestamp=1731369314832Num uniques 1068; 2024-11-11T23:55:14,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369314806, max timestamp=1731369314832Num uniques 1068; 2024-11-11T23:55:14,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369314806, max timestamp=1731369314832Num uniques 1068; 2024-11-11T23:55:14,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T23:55:14,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T23:55:14,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T23:55:14,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731369314866, max timestamp=1731369314906Num uniques 1077; 2024-11-11T23:55:14,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T23:55:14,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731369314866, max timestamp=1731369314906Num uniques 1077; 2024-11-11T23:55:14,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731369314866, max timestamp=1731369314906Num uniques 1077; 2024-11-11T23:55:14,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T23:55:14,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T23:55:14,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731369314957, max timestamp=1731369314973Num uniques 1086; 2024-11-11T23:55:14,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T23:55:14,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731369314957, max timestamp=1731369314973Num uniques 1086; 2024-11-11T23:55:14,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T23:55:14,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731369314957, max timestamp=1731369314973Num uniques 1086; 2024-11-11T23:55:14,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T23:55:15,009 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/c30fd7d740a64e3391a61ab3aa96b653 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/c30fd7d740a64e3391a61ab3aa96b653 2024-11-11T23:55:15,013 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/A of ed59ed5c1dfde989ef4a8e10d39d3e5e into c30fd7d740a64e3391a61ab3aa96b653(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:15,013 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:15,013 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/A, priority=13, startTime=1731369314163; duration=0sec 2024-11-11T23:55:15,013 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:15,013 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:A 2024-11-11T23:55:15,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731369315032, max timestamp=1731369315053Num uniques 1095; 2024-11-11T23:55:15,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-11T23:55:15,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731369315032, max timestamp=1731369315053Num uniques 1095; 2024-11-11T23:55:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-11T23:55:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731369315032, max timestamp=1731369315053Num uniques 1095; 2024-11-11T23:55:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-11T23:55:15,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369375120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:15,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369375121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:15,133 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6b2d5548f0b141e5b211209c73e80db4 2024-11-11T23:55:15,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/ae2d8298148c4d09806c13c3e7cac6a0 is 50, key is test_row_0/B:col10/1731369312523/Put/seqid=0 2024-11-11T23:55:15,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742214_1390 (size=12301) 2024-11-11T23:55:15,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369375235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:15,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369375235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:15,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:15,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369375444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:15,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369375455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:15,541 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/ae2d8298148c4d09806c13c3e7cac6a0 2024-11-11T23:55:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/919b7eae897a4e868a49780464cc2a01 is 50, key is test_row_0/C:col10/1731369312523/Put/seqid=0 2024-11-11T23:55:15,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742215_1391 (size=12301) 2024-11-11T23:55:15,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369375754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369375764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:15,950 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/919b7eae897a4e868a49780464cc2a01 2024-11-11T23:55:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6b2d5548f0b141e5b211209c73e80db4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6b2d5548f0b141e5b211209c73e80db4 2024-11-11T23:55:15,957 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6b2d5548f0b141e5b211209c73e80db4, entries=150, sequenceid=271, filesize=30.5 K 2024-11-11T23:55:15,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/ae2d8298148c4d09806c13c3e7cac6a0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2d8298148c4d09806c13c3e7cac6a0 2024-11-11T23:55:15,961 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2d8298148c4d09806c13c3e7cac6a0, entries=150, sequenceid=271, filesize=12.0 K 2024-11-11T23:55:15,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/919b7eae897a4e868a49780464cc2a01 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/919b7eae897a4e868a49780464cc2a01 2024-11-11T23:55:15,965 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/919b7eae897a4e868a49780464cc2a01, entries=150, sequenceid=271, filesize=12.0 K 2024-11-11T23:55:15,965 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1668ms, sequenceid=271, compaction requested=false 2024-11-11T23:55:15,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:15,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:15,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T23:55:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T23:55:15,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T23:55:15,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3500 sec 2024-11-11T23:55:15,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.3540 sec 2024-11-11T23:55:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:16,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=161.02 KB heapSize=452.63 KB 2024-11-11T23:55:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:16,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d3aeb0414a9c45baae181b9e0365b39e_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369315101/Put/seqid=0 2024-11-11T23:55:16,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742216_1392 (size=14994) 2024-11-11T23:55:16,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369376296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:16,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369376296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:16,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369376404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369376414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369376625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:16,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369376634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:16,692 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:16,695 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d3aeb0414a9c45baae181b9e0365b39e_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3aeb0414a9c45baae181b9e0365b39e_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:16,695 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/1df05aa8184946af9daa2626fdf1d29a, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:16,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/1df05aa8184946af9daa2626fdf1d29a is 175, key is test_row_0/A:col10/1731369315101/Put/seqid=0 2024-11-11T23:55:16,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742217_1393 (size=39949) 2024-11-11T23:55:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T23:55:16,784 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:16,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T23:55:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T23:55:16,786 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:16,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:16,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T23:55:16,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T23:55:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:16,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:16,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369376944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:16,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369376944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:17,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T23:55:17,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:17,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/1df05aa8184946af9daa2626fdf1d29a 2024-11-11T23:55:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T23:55:17,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/cf89ee7d272b458884913b404b427eef is 50, key is test_row_0/B:col10/1731369315101/Put/seqid=0 2024-11-11T23:55:17,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742218_1394 (size=12301) 2024-11-11T23:55:17,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T23:55:17,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T23:55:17,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:17,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T23:55:17,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1731369377464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:17,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731369377464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:17,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/cf89ee7d272b458884913b404b427eef 2024-11-11T23:55:17,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/9ac3eecdff184295ab6258502e9d0cf4 is 50, key is test_row_0/C:col10/1731369315101/Put/seqid=0 2024-11-11T23:55:17,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742219_1395 (size=12301) 2024-11-11T23:55:17,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T23:55:17,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:17,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T23:55:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T23:55:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:17,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/9ac3eecdff184295ab6258502e9d0cf4 2024-11-11T23:55:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/1df05aa8184946af9daa2626fdf1d29a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/1df05aa8184946af9daa2626fdf1d29a 2024-11-11T23:55:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T23:55:17,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/1df05aa8184946af9daa2626fdf1d29a, entries=200, sequenceid=379, filesize=39.0 K 2024-11-11T23:55:17,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/cf89ee7d272b458884913b404b427eef as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cf89ee7d272b458884913b404b427eef 2024-11-11T23:55:17,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cf89ee7d272b458884913b404b427eef, entries=150, sequenceid=379, filesize=12.0 K 2024-11-11T23:55:17,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/9ac3eecdff184295ab6258502e9d0cf4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/9ac3eecdff184295ab6258502e9d0cf4 2024-11-11T23:55:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/9ac3eecdff184295ab6258502e9d0cf4, entries=150, sequenceid=379, filesize=12.0 K 2024-11-11T23:55:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~452.58 KB/463440, currentSize=20.13 KB/20610 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1654ms, sequenceid=379, compaction requested=true 2024-11-11T23:55:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:17,935 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:17,935 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed59ed5c1dfde989ef4a8e10d39d3e5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,936 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:17,936 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:17,936 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/A is initiating minor compaction (all files) 2024-11-11T23:55:17,936 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/B is initiating minor compaction (all files) 2024-11-11T23:55:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,936 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/A in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,936 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/B in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,936 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/c30fd7d740a64e3391a61ab3aa96b653, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6b2d5548f0b141e5b211209c73e80db4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/1df05aa8184946af9daa2626fdf1d29a] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=100.5 K 2024-11-11T23:55:17,936 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e4e645b8081a4fac9b50025f8a93426c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2d8298148c4d09806c13c3e7cac6a0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cf89ee7d272b458884913b404b427eef] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=36.5 K 2024-11-11T23:55:17,936 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,936 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/c30fd7d740a64e3391a61ab3aa96b653, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6b2d5548f0b141e5b211209c73e80db4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/1df05aa8184946af9daa2626fdf1d29a] 2024-11-11T23:55:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,936 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting e4e645b8081a4fac9b50025f8a93426c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731369311321 2024-11-11T23:55:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,937 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30fd7d740a64e3391a61ab3aa96b653, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731369311321 2024-11-11T23:55:17,937 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ae2d8298148c4d09806c13c3e7cac6a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731369312517 2024-11-11T23:55:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,937 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b2d5548f0b141e5b211209c73e80db4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731369312517 2024-11-11T23:55:17,937 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting cf89ee7d272b458884913b404b427eef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731369315101 2024-11-11T23:55:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,937 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df05aa8184946af9daa2626fdf1d29a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731369315101 2024-11-11T23:55:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,942 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,944 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,944 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/c358483dcde548378954f95825e3b5cc is 50, key is test_row_0/B:col10/1731369315101/Put/seqid=0 2024-11-11T23:55:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,946 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115cd161ed862d465da0a5e818e74ac737_ed59ed5c1dfde989ef4a8e10d39d3e5e store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,948 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115cd161ed862d465da0a5e818e74ac737_ed59ed5c1dfde989ef4a8e10d39d3e5e, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:17,948 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115cd161ed862d465da0a5e818e74ac737_ed59ed5c1dfde989ef4a8e10d39d3e5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742220_1396 (size=12847) 2024-11-11T23:55:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,961 INFO [Thread-1530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1530. 2024-11-11T23:55:17,962 DEBUG [Thread-1530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:17,962 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,962 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:17,962 INFO [Thread-1528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1528. 2024-11-11T23:55:17,962 DEBUG [Thread-1528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:17,962 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,962 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742221_1397 (size=4469) 2024-11-11T23:55:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,966 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#A#compaction#337 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,966 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/4116af020a4348e585e31e8eae2aa627 is 175, key is test_row_0/A:col10/1731369315101/Put/seqid=0 2024-11-11T23:55:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,968 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/c358483dcde548378954f95825e3b5cc as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/c358483dcde548378954f95825e3b5cc 2024-11-11T23:55:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,973 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/B of ed59ed5c1dfde989ef4a8e10d39d3e5e into c358483dcde548378954f95825e3b5cc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:17,973 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,973 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/B, priority=13, startTime=1731369317935; duration=0sec 2024-11-11T23:55:17,973 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:17,973 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:B 2024-11-11T23:55:17,973 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,976 INFO [Thread-1526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1526. 2024-11-11T23:55:17,976 DEBUG [Thread-1526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:17,976 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,976 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,978 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,978 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): ed59ed5c1dfde989ef4a8e10d39d3e5e/C is initiating minor compaction (all files) 2024-11-11T23:55:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,978 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed59ed5c1dfde989ef4a8e10d39d3e5e/C in TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:17,978 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/dfc4e9e0bac44acfa2f2923064193872, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/919b7eae897a4e868a49780464cc2a01, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/9ac3eecdff184295ab6258502e9d0cf4] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp, totalSize=36.5 K 2024-11-11T23:55:17,978 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc4e9e0bac44acfa2f2923064193872, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731369311321 2024-11-11T23:55:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 919b7eae897a4e868a49780464cc2a01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731369312517 2024-11-11T23:55:17,979 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac3eecdff184295ab6258502e9d0cf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731369315101 2024-11-11T23:55:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,991 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed59ed5c1dfde989ef4a8e10d39d3e5e#C#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:17,992 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/1871069395b04856a0d2b7d1b6dea156 is 50, key is test_row_0/C:col10/1731369315101/Put/seqid=0 2024-11-11T23:55:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,994 INFO [Thread-1532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1532. 2024-11-11T23:55:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:17,994 DEBUG [Thread-1532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:17,994 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742222_1398 (size=31801) 2024-11-11T23:55:17,994 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:17,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742223_1399 (size=12847) 2024-11-11T23:55:17,998 INFO [Thread-1524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1524. 2024-11-11T23:55:17,998 DEBUG [Thread-1524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:17,998 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,998 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:17,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:18,000 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/4116af020a4348e585e31e8eae2aa627 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/4116af020a4348e585e31e8eae2aa627 2024-11-11T23:55:18,001 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/1871069395b04856a0d2b7d1b6dea156 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1871069395b04856a0d2b7d1b6dea156 2024-11-11T23:55:18,004 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/C of ed59ed5c1dfde989ef4a8e10d39d3e5e into 1871069395b04856a0d2b7d1b6dea156(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:18,004 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed59ed5c1dfde989ef4a8e10d39d3e5e/A of ed59ed5c1dfde989ef4a8e10d39d3e5e into 4116af020a4348e585e31e8eae2aa627(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:18,004 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:18,004 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:18,004 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/A, priority=13, startTime=1731369317935; duration=0sec 2024-11-11T23:55:18,004 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., storeName=ed59ed5c1dfde989ef4a8e10d39d3e5e/C, priority=13, startTime=1731369317935; duration=0sec 2024-11-11T23:55:18,004 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:18,004 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:A 2024-11-11T23:55:18,004 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:18,004 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed59ed5c1dfde989ef4a8e10d39d3e5e:C 2024-11-11T23:55:18,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T23:55:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:18,008 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-11T23:55:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:18,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115410194bfe63420d9f6a7bfc9c156334_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369316294/Put/seqid=0 2024-11-11T23:55:18,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742224_1400 (size=9914) 2024-11-11T23:55:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:18,427 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115410194bfe63420d9f6a7bfc9c156334_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115410194bfe63420d9f6a7bfc9c156334_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:18,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/d2821125fc5e47798682e3b2a89d0e32, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/d2821125fc5e47798682e3b2a89d0e32 is 175, key is test_row_0/A:col10/1731369316294/Put/seqid=0 2024-11-11T23:55:18,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742225_1401 (size=22561) 2024-11-11T23:55:18,488 INFO [Thread-1517 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1517. 2024-11-11T23:55:18,488 INFO [Thread-1515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1515. 2024-11-11T23:55:18,488 DEBUG [Thread-1517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:18,488 DEBUG [Thread-1515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:18,488 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,488 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,488 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,488 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:18,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:18,838 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/d2821125fc5e47798682e3b2a89d0e32 2024-11-11T23:55:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/e6fe25836e07470ba357f280316325f4 is 50, key is test_row_0/B:col10/1731369316294/Put/seqid=0 2024-11-11T23:55:18,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742226_1402 (size=9857) 2024-11-11T23:55:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T23:55:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. as already flushing 2024-11-11T23:55:18,979 INFO [Thread-1513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1513. 2024-11-11T23:55:18,979 DEBUG [Thread-1513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:18,979 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,979 INFO [Thread-1521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1521. 2024-11-11T23:55:18,979 DEBUG [Thread-1521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:18,979 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,979 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:18,980 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:18,998 INFO [Thread-1519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1519. 2024-11-11T23:55:18,998 DEBUG [Thread-1519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:18,998 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,998 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:18,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:19,257 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/e6fe25836e07470ba357f280316325f4 2024-11-11T23:55:19,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/34a18f6bf05c400fad6e742f2064c136 is 50, key is test_row_0/C:col10/1731369316294/Put/seqid=0 2024-11-11T23:55:19,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742227_1403 (size=9857) 2024-11-11T23:55:19,671 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/34a18f6bf05c400fad6e742f2064c136 2024-11-11T23:55:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/d2821125fc5e47798682e3b2a89d0e32 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d2821125fc5e47798682e3b2a89d0e32 2024-11-11T23:55:19,686 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d2821125fc5e47798682e3b2a89d0e32, entries=100, sequenceid=388, filesize=22.0 K 2024-11-11T23:55:19,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/e6fe25836e07470ba357f280316325f4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e6fe25836e07470ba357f280316325f4 2024-11-11T23:55:19,690 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e6fe25836e07470ba357f280316325f4, entries=100, sequenceid=388, filesize=9.6 K 2024-11-11T23:55:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/34a18f6bf05c400fad6e742f2064c136 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/34a18f6bf05c400fad6e742f2064c136 2024-11-11T23:55:19,693 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/34a18f6bf05c400fad6e742f2064c136, entries=100, sequenceid=388, filesize=9.6 K 2024-11-11T23:55:19,694 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=33.54 KB/34350 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1686ms, sequenceid=388, compaction requested=false 2024-11-11T23:55:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: 2024-11-11T23:55:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T23:55:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T23:55:19,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T23:55:19,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9080 sec 2024-11-11T23:55:19,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.9130 sec 2024-11-11T23:55:19,731 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:55:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T23:55:20,955 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:20,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T23:55:20,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-11T23:55:20,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1139 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3416 rows 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1130 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3389 rows 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1141 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3420 rows 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1135 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3402 rows 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-11T23:55:20,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3393 rows 2024-11-11T23:55:20,956 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:55:20,956 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:55:20,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:20,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:20,957 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:20,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T23:55:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T23:55:20,965 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369320965"}]},"ts":"1731369320965"} 2024-11-11T23:55:20,967 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T23:55:20,967 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T23:55:20,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:55:20,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, UNASSIGN}] 2024-11-11T23:55:20,969 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, UNASSIGN 2024-11-11T23:55:20,970 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=ed59ed5c1dfde989ef4a8e10d39d3e5e, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:20,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, UNASSIGN because future has completed 2024-11-11T23:55:20,971 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:55:20,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:55:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T23:55:21,124 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:21,125 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:55:21,125 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing ed59ed5c1dfde989ef4a8e10d39d3e5e, disabling compactions & flushes 2024-11-11T23:55:21,125 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:21,125 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:21,125 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. after waiting 0 ms 2024-11-11T23:55:21,125 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:21,126 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T23:55:21,126 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=A 2024-11-11T23:55:21,126 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:21,126 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=B 2024-11-11T23:55:21,127 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:21,127 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed59ed5c1dfde989ef4a8e10d39d3e5e, store=C 2024-11-11T23:55:21,127 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:21,138 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111193b7c4ef9e614ed9b44b4e870489a623_ed59ed5c1dfde989ef4a8e10d39d3e5e is 50, key is test_row_0/A:col10/1731369318995/Put/seqid=0 2024-11-11T23:55:21,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742228_1404 (size=12454) 2024-11-11T23:55:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T23:55:21,542 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:21,552 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111193b7c4ef9e614ed9b44b4e870489a623_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193b7c4ef9e614ed9b44b4e870489a623_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:21,554 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6e61506244d34d9ab0423357b8a1e624, store: [table=TestAcidGuarantees family=A region=ed59ed5c1dfde989ef4a8e10d39d3e5e] 2024-11-11T23:55:21,555 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6e61506244d34d9ab0423357b8a1e624 is 175, key is test_row_0/A:col10/1731369318995/Put/seqid=0 2024-11-11T23:55:21,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742229_1405 (size=31255) 2024-11-11T23:55:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T23:55:21,961 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6e61506244d34d9ab0423357b8a1e624 2024-11-11T23:55:21,973 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/81abb2ba3d604b7fbda83b2959d2693c is 50, key is test_row_0/B:col10/1731369318995/Put/seqid=0 2024-11-11T23:55:21,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742230_1406 (size=12301) 2024-11-11T23:55:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T23:55:22,378 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/81abb2ba3d604b7fbda83b2959d2693c 2024-11-11T23:55:22,389 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/784d08c78e4d47d6a425befbb53ded24 is 50, key is test_row_0/C:col10/1731369318995/Put/seqid=0 2024-11-11T23:55:22,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742231_1407 (size=12301) 2024-11-11T23:55:22,794 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/784d08c78e4d47d6a425befbb53ded24 2024-11-11T23:55:22,805 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/A/6e61506244d34d9ab0423357b8a1e624 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6e61506244d34d9ab0423357b8a1e624 2024-11-11T23:55:22,810 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6e61506244d34d9ab0423357b8a1e624, entries=150, sequenceid=396, filesize=30.5 K 2024-11-11T23:55:22,811 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/B/81abb2ba3d604b7fbda83b2959d2693c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/81abb2ba3d604b7fbda83b2959d2693c 2024-11-11T23:55:22,815 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/81abb2ba3d604b7fbda83b2959d2693c, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T23:55:22,816 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/.tmp/C/784d08c78e4d47d6a425befbb53ded24 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/784d08c78e4d47d6a425befbb53ded24 2024-11-11T23:55:22,819 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/784d08c78e4d47d6a425befbb53ded24, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T23:55:22,820 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1695ms, sequenceid=396, compaction requested=true 2024-11-11T23:55:22,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/2171e88a947b48a68c9206e8469beda5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/a9ccacf3d92141f3aa843f097519f744, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/0f6a2c87c993497a9efe4d11c4d742d0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/37a501921ff64ad7a430bb8f80fb65e8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6459fd08c74a4a3b8c29cbd9279221f0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/ef956f0de1be41dc8c2be7366781706b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/7882b59f00fa40cda96e3fd4a7bf1052, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/3a023efaab064d48a832a66ed559d086, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/8f050526188b4117a1f2351a28593316, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/203b85e45fe040389e3a7593158a869c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/18ff3a470e6d4db59b04057a6ee0ddbd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/809db127b29a45a09a945bfccc34a556, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d3b9e6f9f1d84149a15a1d88d0f634cf, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/c30fd7d740a64e3391a61ab3aa96b653, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6b2d5548f0b141e5b211209c73e80db4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/1df05aa8184946af9daa2626fdf1d29a] to archive 2024-11-11T23:55:22,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:55:22,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/2171e88a947b48a68c9206e8469beda5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/2171e88a947b48a68c9206e8469beda5 2024-11-11T23:55:22,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/a9ccacf3d92141f3aa843f097519f744 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/a9ccacf3d92141f3aa843f097519f744 2024-11-11T23:55:22,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/0f6a2c87c993497a9efe4d11c4d742d0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/0f6a2c87c993497a9efe4d11c4d742d0 2024-11-11T23:55:22,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/37a501921ff64ad7a430bb8f80fb65e8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/37a501921ff64ad7a430bb8f80fb65e8 2024-11-11T23:55:22,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6459fd08c74a4a3b8c29cbd9279221f0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6459fd08c74a4a3b8c29cbd9279221f0 2024-11-11T23:55:22,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/ef956f0de1be41dc8c2be7366781706b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/ef956f0de1be41dc8c2be7366781706b 2024-11-11T23:55:22,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/7882b59f00fa40cda96e3fd4a7bf1052 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/7882b59f00fa40cda96e3fd4a7bf1052 2024-11-11T23:55:22,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/3a023efaab064d48a832a66ed559d086 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/3a023efaab064d48a832a66ed559d086 2024-11-11T23:55:22,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/8f050526188b4117a1f2351a28593316 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/8f050526188b4117a1f2351a28593316 2024-11-11T23:55:22,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/203b85e45fe040389e3a7593158a869c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/203b85e45fe040389e3a7593158a869c 2024-11-11T23:55:22,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/18ff3a470e6d4db59b04057a6ee0ddbd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/18ff3a470e6d4db59b04057a6ee0ddbd 2024-11-11T23:55:22,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/809db127b29a45a09a945bfccc34a556 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/809db127b29a45a09a945bfccc34a556 2024-11-11T23:55:22,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d3b9e6f9f1d84149a15a1d88d0f634cf to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d3b9e6f9f1d84149a15a1d88d0f634cf 2024-11-11T23:55:22,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/c30fd7d740a64e3391a61ab3aa96b653 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/c30fd7d740a64e3391a61ab3aa96b653 2024-11-11T23:55:22,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6b2d5548f0b141e5b211209c73e80db4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6b2d5548f0b141e5b211209c73e80db4 2024-11-11T23:55:22,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/1df05aa8184946af9daa2626fdf1d29a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/1df05aa8184946af9daa2626fdf1d29a 2024-11-11T23:55:22,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/10bc72e24cb64066b90714e3851eafd5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/9dcd1ba5f11f4cc9839b18d564929c51, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/388280427d7c49629a2661823c354aff, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/77ec39faae8140f2a67f788a38c66882, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ac3c263d92a9406ebaefbee8d910ce92, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aafc451ecad4f1fa258792b0a986416, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2124ecaf054e4fbc380e206dac01c5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0949c437b2f44ea7a69ddce1412c6f90, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cdbe8e9396134913a2d898a98cad1a89, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/6a03b656b2544c2fb3c79e94e8c12c81, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/85ee8d80d90c410586bdcc03b63c5de5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aa3a643cb4e42aaa91e0f448e8e5d36, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e4e645b8081a4fac9b50025f8a93426c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/266765c7a7a6421d987f7b896abdab9a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2d8298148c4d09806c13c3e7cac6a0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cf89ee7d272b458884913b404b427eef] to archive 2024-11-11T23:55:22,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:55:22,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/10bc72e24cb64066b90714e3851eafd5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/10bc72e24cb64066b90714e3851eafd5 2024-11-11T23:55:22,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/9dcd1ba5f11f4cc9839b18d564929c51 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/9dcd1ba5f11f4cc9839b18d564929c51 2024-11-11T23:55:22,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/388280427d7c49629a2661823c354aff to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/388280427d7c49629a2661823c354aff 2024-11-11T23:55:22,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/77ec39faae8140f2a67f788a38c66882 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/77ec39faae8140f2a67f788a38c66882 2024-11-11T23:55:22,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ac3c263d92a9406ebaefbee8d910ce92 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ac3c263d92a9406ebaefbee8d910ce92 2024-11-11T23:55:22,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aafc451ecad4f1fa258792b0a986416 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aafc451ecad4f1fa258792b0a986416 2024-11-11T23:55:22,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2124ecaf054e4fbc380e206dac01c5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2124ecaf054e4fbc380e206dac01c5 2024-11-11T23:55:22,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0949c437b2f44ea7a69ddce1412c6f90 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0949c437b2f44ea7a69ddce1412c6f90 2024-11-11T23:55:22,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cdbe8e9396134913a2d898a98cad1a89 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cdbe8e9396134913a2d898a98cad1a89 2024-11-11T23:55:22,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/6a03b656b2544c2fb3c79e94e8c12c81 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/6a03b656b2544c2fb3c79e94e8c12c81 2024-11-11T23:55:22,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/85ee8d80d90c410586bdcc03b63c5de5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/85ee8d80d90c410586bdcc03b63c5de5 2024-11-11T23:55:22,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aa3a643cb4e42aaa91e0f448e8e5d36 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/0aa3a643cb4e42aaa91e0f448e8e5d36 2024-11-11T23:55:22,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e4e645b8081a4fac9b50025f8a93426c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e4e645b8081a4fac9b50025f8a93426c 2024-11-11T23:55:22,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/266765c7a7a6421d987f7b896abdab9a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/266765c7a7a6421d987f7b896abdab9a 2024-11-11T23:55:22,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2d8298148c4d09806c13c3e7cac6a0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/ae2d8298148c4d09806c13c3e7cac6a0 2024-11-11T23:55:22,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cf89ee7d272b458884913b404b427eef to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/cf89ee7d272b458884913b404b427eef 2024-11-11T23:55:22,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/b91a59dbbe2c4134a8653ce2262723f4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/86bb4b69fb0e468186c8d842e433a903, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a642ecc595674a91a64fdf1e55ffa5e1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/fbbf996958034b8c80cef599a96a7640, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/e3af6aed99d54f7283f2883b5c9efafd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1d51b3c7a589423385d6d7a9daf9bf14, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6c0b3743ceac4c81885576a1bd65d9c8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6925f052fe164291a4d30250a03ee079, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a80360bc08114be9ba5eddde515f8254, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/f72c3af867384a94b31a48afc789fe9d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6a3113ab3b4c4055976843f7b12ff414, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/4b6fb97cdd664a18b6c44b527782f64e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/dfc4e9e0bac44acfa2f2923064193872, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/c31bcdf458964c3886bbea3e889cf84a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/919b7eae897a4e868a49780464cc2a01, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/9ac3eecdff184295ab6258502e9d0cf4] to archive 2024-11-11T23:55:22,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:55:22,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/b91a59dbbe2c4134a8653ce2262723f4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/b91a59dbbe2c4134a8653ce2262723f4 2024-11-11T23:55:22,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/86bb4b69fb0e468186c8d842e433a903 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/86bb4b69fb0e468186c8d842e433a903 2024-11-11T23:55:22,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a642ecc595674a91a64fdf1e55ffa5e1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a642ecc595674a91a64fdf1e55ffa5e1 2024-11-11T23:55:22,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/fbbf996958034b8c80cef599a96a7640 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/fbbf996958034b8c80cef599a96a7640 2024-11-11T23:55:22,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/e3af6aed99d54f7283f2883b5c9efafd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/e3af6aed99d54f7283f2883b5c9efafd 2024-11-11T23:55:22,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1d51b3c7a589423385d6d7a9daf9bf14 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1d51b3c7a589423385d6d7a9daf9bf14 2024-11-11T23:55:22,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6c0b3743ceac4c81885576a1bd65d9c8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6c0b3743ceac4c81885576a1bd65d9c8 2024-11-11T23:55:22,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6925f052fe164291a4d30250a03ee079 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6925f052fe164291a4d30250a03ee079 2024-11-11T23:55:22,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a80360bc08114be9ba5eddde515f8254 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/a80360bc08114be9ba5eddde515f8254 2024-11-11T23:55:22,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/f72c3af867384a94b31a48afc789fe9d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/f72c3af867384a94b31a48afc789fe9d 2024-11-11T23:55:22,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6a3113ab3b4c4055976843f7b12ff414 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/6a3113ab3b4c4055976843f7b12ff414 2024-11-11T23:55:22,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/4b6fb97cdd664a18b6c44b527782f64e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/4b6fb97cdd664a18b6c44b527782f64e 2024-11-11T23:55:22,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/dfc4e9e0bac44acfa2f2923064193872 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/dfc4e9e0bac44acfa2f2923064193872 2024-11-11T23:55:22,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/c31bcdf458964c3886bbea3e889cf84a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/c31bcdf458964c3886bbea3e889cf84a 2024-11-11T23:55:22,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/919b7eae897a4e868a49780464cc2a01 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/919b7eae897a4e868a49780464cc2a01 2024-11-11T23:55:22,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/9ac3eecdff184295ab6258502e9d0cf4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/9ac3eecdff184295ab6258502e9d0cf4 2024-11-11T23:55:22,878 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/recovered.edits/399.seqid, newMaxSeqId=399, maxSeqId=4 2024-11-11T23:55:22,879 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. 2024-11-11T23:55:22,879 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for ed59ed5c1dfde989ef4a8e10d39d3e5e: Waiting for close lock at 1731369321125Running coprocessor pre-close hooks at 1731369321125Disabling compacts and flushes for region at 1731369321125Disabling writes for close at 1731369321125Obtaining lock to block concurrent updates at 1731369321126 (+1 ms)Preparing flush snapshotting stores in ed59ed5c1dfde989ef4a8e10d39d3e5e at 1731369321126Finished memstore snapshotting TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731369321127 (+1 ms)Flushing stores of TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e. at 1731369321128 (+1 ms)Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/A: creating writer at 1731369321128Flushing mob file ed59ed5c1dfde989ef4a8e10d39d3e5e/A: appending metadata at 1731369321138 (+10 ms)Flushing mob file ed59ed5c1dfde989ef4a8e10d39d3e5e/A: closing flushed file at 1731369321138Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/A: appending metadata at 1731369321554 (+416 ms)Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/A: closing flushed file at 1731369321554Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/B: creating writer at 1731369321969 (+415 ms)Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/B: appending metadata at 1731369321972 (+3 ms)Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/B: closing flushed file at 1731369321972Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/C: creating writer at 1731369322386 (+414 ms)Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/C: appending metadata at 1731369322388 (+2 ms)Flushing ed59ed5c1dfde989ef4a8e10d39d3e5e/C: closing flushed file at 1731369322388Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f7166b1: reopening flushed file at 1731369322804 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@681125ec: reopening flushed file at 1731369322810 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@523f0a65: reopening flushed file at 1731369322815 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ed59ed5c1dfde989ef4a8e10d39d3e5e in 1695ms, sequenceid=396, compaction requested=true at 1731369322820 (+5 ms)Writing region close event to WAL at 1731369322876 (+56 ms)Running coprocessor post-close hooks at 1731369322878 (+2 ms)Closed at 1731369322879 (+1 ms) 2024-11-11T23:55:22,880 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:22,881 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=ed59ed5c1dfde989ef4a8e10d39d3e5e, regionState=CLOSED 2024-11-11T23:55:22,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:55:22,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T23:55:22,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure ed59ed5c1dfde989ef4a8e10d39d3e5e, server=b70d29849faf,42393,1731369201430 in 1.9120 sec 2024-11-11T23:55:22,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-11T23:55:22,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed59ed5c1dfde989ef4a8e10d39d3e5e, UNASSIGN in 1.9160 sec 2024-11-11T23:55:22,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T23:55:22,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9190 sec 2024-11-11T23:55:22,889 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369322889"}]},"ts":"1731369322889"} 2024-11-11T23:55:22,890 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T23:55:22,890 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T23:55:22,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9320 sec 2024-11-11T23:55:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T23:55:23,115 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:23,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T23:55:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:23,119 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T23:55:23,120 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:23,124 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,127 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/recovered.edits] 2024-11-11T23:55:23,131 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/4116af020a4348e585e31e8eae2aa627 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/4116af020a4348e585e31e8eae2aa627 2024-11-11T23:55:23,132 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6e61506244d34d9ab0423357b8a1e624 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/6e61506244d34d9ab0423357b8a1e624 2024-11-11T23:55:23,134 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d2821125fc5e47798682e3b2a89d0e32 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/A/d2821125fc5e47798682e3b2a89d0e32 2024-11-11T23:55:23,137 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/81abb2ba3d604b7fbda83b2959d2693c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/81abb2ba3d604b7fbda83b2959d2693c 2024-11-11T23:55:23,138 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/c358483dcde548378954f95825e3b5cc to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/c358483dcde548378954f95825e3b5cc 2024-11-11T23:55:23,139 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e6fe25836e07470ba357f280316325f4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/B/e6fe25836e07470ba357f280316325f4 2024-11-11T23:55:23,142 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1871069395b04856a0d2b7d1b6dea156 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/1871069395b04856a0d2b7d1b6dea156 2024-11-11T23:55:23,143 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/34a18f6bf05c400fad6e742f2064c136 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/34a18f6bf05c400fad6e742f2064c136 2024-11-11T23:55:23,144 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/784d08c78e4d47d6a425befbb53ded24 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/C/784d08c78e4d47d6a425befbb53ded24 2024-11-11T23:55:23,146 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/recovered.edits/399.seqid to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e/recovered.edits/399.seqid 2024-11-11T23:55:23,146 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,146 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T23:55:23,146 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T23:55:23,147 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T23:55:23,149 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146b51a98773a4ee298a580549f061379_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146b51a98773a4ee298a580549f061379_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115410194bfe63420d9f6a7bfc9c156334_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115410194bfe63420d9f6a7bfc9c156334_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111759a0ae20df143649df82b9bc0981218_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111759a0ae20df143649df82b9bc0981218_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,151 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117acde49e09104f1daed819cab3520cf2_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117acde49e09104f1daed819cab3520cf2_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,152 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117eac9306888f457298ed3329a82da2dc_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117eac9306888f457298ed3329a82da2dc_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,153 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192d9a991728646e9ab198fead9900373_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192d9a991728646e9ab198fead9900373_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,153 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193b7c4ef9e614ed9b44b4e870489a623_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193b7c4ef9e614ed9b44b4e870489a623_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,154 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119abeaa945fa94f448f827911f3e733df_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119abeaa945fa94f448f827911f3e733df_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,155 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a4c18988bd4e43bc9d303905beb31785_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a4c18988bd4e43bc9d303905beb31785_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,155 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b8c5022ee7f64b3c9188c4cda27c42ef_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b8c5022ee7f64b3c9188c4cda27c42ef_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ba96436c65434fc780c57e57bc68588c_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ba96436c65434fc780c57e57bc68588c_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,157 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3aeb0414a9c45baae181b9e0365b39e_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3aeb0414a9c45baae181b9e0365b39e_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,158 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df322d02adec4bfda47bddbf1ec038d2_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df322d02adec4bfda47bddbf1ec038d2_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,158 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f85c51f467d74af39286b5075ed347c6_ed59ed5c1dfde989ef4a8e10d39d3e5e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f85c51f467d74af39286b5075ed347c6_ed59ed5c1dfde989ef4a8e10d39d3e5e 2024-11-11T23:55:23,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T23:55:23,160 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:23,162 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T23:55:23,164 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T23:55:23,164 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:23,165 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T23:55:23,165 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731369323165"}]},"ts":"9223372036854775807"} 2024-11-11T23:55:23,166 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T23:55:23,166 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ed59ed5c1dfde989ef4a8e10d39d3e5e, NAME => 'TestAcidGuarantees,,1731369294808.ed59ed5c1dfde989ef4a8e10d39d3e5e.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T23:55:23,166 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T23:55:23,167 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731369323166"}]},"ts":"9223372036854775807"} 2024-11-11T23:55:23,168 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T23:55:23,169 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:23,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-11T23:55:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T23:55:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T23:55:23,228 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:23,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1248936657_22 at /127.0.0.1:40344 [Waiting for operation #898] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1248936657_22 at /127.0.0.1:40360 [Waiting for operation #862] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_809876577_22 at /127.0.0.1:56788 [Waiting for operation #1288] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_809876577_22 at /127.0.0.1:44238 [Waiting for operation #946] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=465 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 474), ProcessCount=11 (was 11), AvailableMemoryMB=4586 (was 4640) 2024-11-11T23:55:23,247 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=4586 2024-11-11T23:55:23,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:55:23,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:23,251 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:23,251 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:23,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-11T23:55:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T23:55:23,252 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:23,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742232_1408 (size=1061) 2024-11-11T23:55:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T23:55:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T23:55:23,664 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 2024-11-11T23:55:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742233_1409 (size=53) 2024-11-11T23:55:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T23:55:24,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:24,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6818f1e02fff6c00ff92806be77a8a11, disabling compactions & flushes 2024-11-11T23:55:24,079 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:24,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:24,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. after waiting 0 ms 2024-11-11T23:55:24,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:24,080 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:24,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6818f1e02fff6c00ff92806be77a8a11: Waiting for close lock at 1731369324079Disabling compacts and flushes for region at 1731369324079Disabling writes for close at 1731369324080 (+1 ms)Writing region close event to WAL at 1731369324080Closed at 1731369324080 2024-11-11T23:55:24,083 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:24,084 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731369324083"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369324083"}]},"ts":"1731369324083"} 2024-11-11T23:55:24,086 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:24,087 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:24,087 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369324087"}]},"ts":"1731369324087"} 2024-11-11T23:55:24,089 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T23:55:24,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, ASSIGN}] 2024-11-11T23:55:24,091 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, ASSIGN 2024-11-11T23:55:24,091 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, ASSIGN; state=OFFLINE, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=false 2024-11-11T23:55:24,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=6818f1e02fff6c00ff92806be77a8a11, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:24,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, ASSIGN because future has completed 2024-11-11T23:55:24,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:55:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T23:55:24,405 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:24,406 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:24,407 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,407 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:24,408 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,408 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,410 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,412 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:24,412 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6818f1e02fff6c00ff92806be77a8a11 columnFamilyName A 2024-11-11T23:55:24,412 DEBUG [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:24,413 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.HStore(327): Store=6818f1e02fff6c00ff92806be77a8a11/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:24,413 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,415 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:24,416 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6818f1e02fff6c00ff92806be77a8a11 columnFamilyName B 2024-11-11T23:55:24,416 DEBUG [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:24,416 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.HStore(327): Store=6818f1e02fff6c00ff92806be77a8a11/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:24,417 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,418 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:24,418 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6818f1e02fff6c00ff92806be77a8a11 columnFamilyName C 2024-11-11T23:55:24,418 DEBUG [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:24,418 INFO [StoreOpener-6818f1e02fff6c00ff92806be77a8a11-1 {}] regionserver.HStore(327): Store=6818f1e02fff6c00ff92806be77a8a11/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:24,418 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:24,418 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,419 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,419 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,420 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,420 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,420 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:55:24,421 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,423 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:24,423 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 6818f1e02fff6c00ff92806be77a8a11; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63306631, jitterRate=-0.056657686829566956}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:55:24,423 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:24,424 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 6818f1e02fff6c00ff92806be77a8a11: Running coprocessor pre-open hook at 1731369324408Writing region info on filesystem at 1731369324408Initializing all the Stores at 1731369324409 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369324409Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369324409Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369324409Cleaning up temporary data from old regions at 1731369324420 (+11 ms)Running coprocessor post-open hooks at 1731369324423 (+3 ms)Region opened successfully at 1731369324424 (+1 ms) 2024-11-11T23:55:24,425 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., pid=110, masterSystemTime=1731369324402 2024-11-11T23:55:24,426 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:24,426 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:24,427 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=6818f1e02fff6c00ff92806be77a8a11, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:24,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:55:24,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T23:55:24,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 in 181 msec 2024-11-11T23:55:24,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-11T23:55:24,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, ASSIGN in 341 msec 2024-11-11T23:55:24,432 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:24,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369324432"}]},"ts":"1731369324432"} 2024-11-11T23:55:24,433 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T23:55:24,434 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:24,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-11T23:55:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T23:55:25,396 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:25,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,401 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6d1014, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,402 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5ca189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,403 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,406 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T23:55:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b70d29849faf,37327,1731369200577 2024-11-11T23:55:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76b35d19 2024-11-11T23:55:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T23:55:25,408 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T23:55:25,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,410 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7abd2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,411 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da19a70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,413 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c89411c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,414 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e988a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,416 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28aee95e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,417 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eb99cd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,419 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a4e4dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,420 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab33e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,422 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56919fe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,423 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dbafcec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,425 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fe3fe58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,426 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f38307, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,428 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23370402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,429 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd0ae54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,431 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37feb0b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448e1e51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,433 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f2dbed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2861fe30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:25,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:25,436 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e534a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:25,438 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0cf19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:25,439 DEBUG [Thread-1854 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,439 DEBUG [Thread-1862 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,439 DEBUG [Thread-1867 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:25,439 DEBUG [Thread-1871 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,440 DEBUG [Thread-1858 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,440 DEBUG [Thread-1869 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,440 DEBUG [Thread-1860 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,440 DEBUG [Thread-1869 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,440 DEBUG [Thread-1869 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,441 DEBUG [Thread-1873 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-11T23:55:25,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T23:55:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,445 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,445 DEBUG [Thread-1856 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,447 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:25,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:25,447 DEBUG [Thread-1865 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:25,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,448 DEBUG [Thread-1865 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:25,448 DEBUG [Thread-1865 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:25,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2] 2024-11-11T23:55:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:25,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:55:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:25,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/83a5ea4fefd841f9903cf36ebdefd486 is 50, key is test_row_0/A:col10/1731369325476/Put/seqid=0 2024-11-11T23:55:25,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742234_1410 (size=16681) 2024-11-11T23:55:25,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/83a5ea4fefd841f9903cf36ebdefd486 2024-11-11T23:55:25,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369385528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731369325509, max timestamp=1731369325535Num uniques 859; 2024-11-11T23:55:25,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731369325509, max timestamp=1731369325535Num uniques 859; 2024-11-11T23:55:25,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T23:55:25,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T23:55:25,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731369325509, max timestamp=1731369325535Num uniques 859; 2024-11-11T23:55:25,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4f2511ccef55406c9524abf98145019d is 50, key is test_row_0/B:col10/1731369325476/Put/seqid=0 2024-11-11T23:55:25,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T23:55:25,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742235_1411 (size=12001) 2024-11-11T23:55:25,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4f2511ccef55406c9524abf98145019d 2024-11-11T23:55:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T23:55:25,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/a797629be6624eb497ba45b23a2ee4d5 is 50, key is test_row_0/C:col10/1731369325476/Put/seqid=0 2024-11-11T23:55:25,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742236_1412 (size=12001) 2024-11-11T23:55:25,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/a797629be6624eb497ba45b23a2ee4d5 2024-11-11T23:55:25,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369385568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369385569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369385570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369385575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/83a5ea4fefd841f9903cf36ebdefd486 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/83a5ea4fefd841f9903cf36ebdefd486 2024-11-11T23:55:25,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/83a5ea4fefd841f9903cf36ebdefd486, entries=250, sequenceid=12, filesize=16.3 K 2024-11-11T23:55:25,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4f2511ccef55406c9524abf98145019d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f2511ccef55406c9524abf98145019d 2024-11-11T23:55:25,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f2511ccef55406c9524abf98145019d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T23:55:25,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/a797629be6624eb497ba45b23a2ee4d5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a797629be6624eb497ba45b23a2ee4d5 2024-11-11T23:55:25,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T23:55:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:25,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a797629be6624eb497ba45b23a2ee4d5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T23:55:25,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6818f1e02fff6c00ff92806be77a8a11 in 136ms, sequenceid=12, compaction requested=false 2024-11-11T23:55:25,616 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T23:55:25,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:25,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-11T23:55:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:25,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:25,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:25,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e185dc45db784027a260a8d0cb6e5c1c is 50, key is test_row_0/A:col10/1731369325652/Put/seqid=0 2024-11-11T23:55:25,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742237_1413 (size=14341) 2024-11-11T23:55:25,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e185dc45db784027a260a8d0cb6e5c1c 2024-11-11T23:55:25,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/f99c16412bac4381b342d0334dff85ac is 50, key is test_row_0/B:col10/1731369325652/Put/seqid=0 2024-11-11T23:55:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742238_1414 (size=12001) 2024-11-11T23:55:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369385685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369385692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369385694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369385696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369385697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T23:55:25,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T23:55:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:25,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369385804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369385814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369385814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369385814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:25,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T23:55:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:25,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:25,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369385904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369386014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369386024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369386024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369386024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T23:55:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:26,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:26,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T23:55:26,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/f99c16412bac4381b342d0334dff85ac 2024-11-11T23:55:26,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/59c7ac7567e34fca991bc3001e030a81 is 50, key is test_row_0/C:col10/1731369325652/Put/seqid=0 2024-11-11T23:55:26,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742239_1415 (size=12001) 2024-11-11T23:55:26,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/59c7ac7567e34fca991bc3001e030a81 2024-11-11T23:55:26,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e185dc45db784027a260a8d0cb6e5c1c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e185dc45db784027a260a8d0cb6e5c1c 2024-11-11T23:55:26,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e185dc45db784027a260a8d0cb6e5c1c, entries=200, sequenceid=46, filesize=14.0 K 2024-11-11T23:55:26,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/f99c16412bac4381b342d0334dff85ac as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f99c16412bac4381b342d0334dff85ac 2024-11-11T23:55:26,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f99c16412bac4381b342d0334dff85ac, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T23:55:26,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/59c7ac7567e34fca991bc3001e030a81 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/59c7ac7567e34fca991bc3001e030a81 2024-11-11T23:55:26,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/59c7ac7567e34fca991bc3001e030a81, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T23:55:26,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for 6818f1e02fff6c00ff92806be77a8a11 in 495ms, sequenceid=46, compaction requested=false 2024-11-11T23:55:26,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:26,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T23:55:26,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:26,213 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T23:55:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:26,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/d7f6c7ba87e34468a81b8907fd9dbc25 is 50, key is test_row_0/A:col10/1731369325692/Put/seqid=0 2024-11-11T23:55:26,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742240_1416 (size=12001) 2024-11-11T23:55:26,225 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/d7f6c7ba87e34468a81b8907fd9dbc25 2024-11-11T23:55:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:26,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/9bcf71eeab2742b596184f458a65fb6b is 50, key is test_row_0/B:col10/1731369325692/Put/seqid=0 2024-11-11T23:55:26,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742241_1417 (size=12001) 2024-11-11T23:55:26,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369326319, max timestamp=1731369326345Num uniques 859; 2024-11-11T23:55:26,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369326319, max timestamp=1731369326345Num uniques 859; 2024-11-11T23:55:26,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:26,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:26,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369326319, max timestamp=1731369326345Num uniques 859; 2024-11-11T23:55:26,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:26,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369386370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369386378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369386379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369386379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369386381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369386484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369386494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369386494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369386494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369386494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T23:55:26,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/9bcf71eeab2742b596184f458a65fb6b 2024-11-11T23:55:26,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/bf567b4e6420493b84568da50ecabf70 is 50, key is test_row_0/C:col10/1731369325692/Put/seqid=0 2024-11-11T23:55:26,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742242_1418 (size=12001) 2024-11-11T23:55:26,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369386704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369386704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369386704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369386704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369386705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369387014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369387014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369387014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369387014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369387014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,045 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/bf567b4e6420493b84568da50ecabf70 2024-11-11T23:55:27,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/d7f6c7ba87e34468a81b8907fd9dbc25 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/d7f6c7ba87e34468a81b8907fd9dbc25 2024-11-11T23:55:27,051 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/d7f6c7ba87e34468a81b8907fd9dbc25, entries=150, sequenceid=57, filesize=11.7 K 2024-11-11T23:55:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/9bcf71eeab2742b596184f458a65fb6b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9bcf71eeab2742b596184f458a65fb6b 2024-11-11T23:55:27,054 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9bcf71eeab2742b596184f458a65fb6b, entries=150, sequenceid=57, filesize=11.7 K 2024-11-11T23:55:27,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/bf567b4e6420493b84568da50ecabf70 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/bf567b4e6420493b84568da50ecabf70 2024-11-11T23:55:27,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/bf567b4e6420493b84568da50ecabf70, entries=150, sequenceid=57, filesize=11.7 K 2024-11-11T23:55:27,058 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6818f1e02fff6c00ff92806be77a8a11 in 845ms, sequenceid=57, compaction requested=true 2024-11-11T23:55:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-11T23:55:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-11T23:55:27,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T23:55:27,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6120 sec 2024-11-11T23:55:27,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.6230 sec 2024-11-11T23:55:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:27,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T23:55:27,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:27,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:27,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:27,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:27,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:27,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:27,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/1982003f109c4138aad4ee30aa718580 is 50, key is test_row_0/A:col10/1731369327539/Put/seqid=0 2024-11-11T23:55:27,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742243_1419 (size=14341) 2024-11-11T23:55:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369387543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369387544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369387550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369387557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369387557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T23:55:27,584 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:27,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-11T23:55:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T23:55:27,586 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:27,587 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:27,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:27,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369387665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369387665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369387665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369387665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369387665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T23:55:27,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T23:55:27,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:27,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:27,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:27,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:27,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:27,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369387875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369387875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T23:55:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:27,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:27,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369387884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369387884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369387884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T23:55:27,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/1982003f109c4138aad4ee30aa718580 2024-11-11T23:55:27,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/3623232015304dae91313284a6c782eb is 50, key is test_row_0/B:col10/1731369327539/Put/seqid=0 2024-11-11T23:55:27,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742244_1420 (size=12001) 2024-11-11T23:55:28,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T23:55:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T23:55:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369388195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369388195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369388204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369388204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369388204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T23:55:28,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T23:55:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/3623232015304dae91313284a6c782eb 2024-11-11T23:55:28,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/9900e905326a4307b842647bd16ba364 is 50, key is test_row_0/C:col10/1731369327539/Put/seqid=0 2024-11-11T23:55:28,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742245_1421 (size=12001) 2024-11-11T23:55:28,400 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:55:28,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T23:55:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T23:55:28,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:28,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369388715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369388715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369388715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369388715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T23:55:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369388724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:28,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/9900e905326a4307b842647bd16ba364 2024-11-11T23:55:28,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/1982003f109c4138aad4ee30aa718580 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1982003f109c4138aad4ee30aa718580 2024-11-11T23:55:28,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1982003f109c4138aad4ee30aa718580, entries=200, sequenceid=93, filesize=14.0 K 2024-11-11T23:55:28,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/3623232015304dae91313284a6c782eb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/3623232015304dae91313284a6c782eb 2024-11-11T23:55:28,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/3623232015304dae91313284a6c782eb, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T23:55:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/9900e905326a4307b842647bd16ba364 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9900e905326a4307b842647bd16ba364 2024-11-11T23:55:28,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9900e905326a4307b842647bd16ba364, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T23:55:28,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6818f1e02fff6c00ff92806be77a8a11 in 1239ms, sequenceid=93, compaction requested=true 2024-11-11T23:55:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:28,779 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:28,779 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:28,780 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:28,780 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:28,780 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:28,780 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:28,780 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,780 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,780 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f2511ccef55406c9524abf98145019d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f99c16412bac4381b342d0334dff85ac, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9bcf71eeab2742b596184f458a65fb6b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/3623232015304dae91313284a6c782eb] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=46.9 K 2024-11-11T23:55:28,780 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/83a5ea4fefd841f9903cf36ebdefd486, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e185dc45db784027a260a8d0cb6e5c1c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/d7f6c7ba87e34468a81b8907fd9dbc25, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1982003f109c4138aad4ee30aa718580] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=56.0 K 2024-11-11T23:55:28,781 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a5ea4fefd841f9903cf36ebdefd486, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369325445 2024-11-11T23:55:28,781 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2511ccef55406c9524abf98145019d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369325476 2024-11-11T23:55:28,781 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f99c16412bac4381b342d0334dff85ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731369325568 2024-11-11T23:55:28,781 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e185dc45db784027a260a8d0cb6e5c1c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731369325565 2024-11-11T23:55:28,781 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bcf71eeab2742b596184f458a65fb6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731369325691 2024-11-11T23:55:28,781 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7f6c7ba87e34468a81b8907fd9dbc25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731369325691 2024-11-11T23:55:28,781 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3623232015304dae91313284a6c782eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731369326377 2024-11-11T23:55:28,781 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1982003f109c4138aad4ee30aa718580, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731369326377 2024-11-11T23:55:28,789 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:28,789 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:28,789 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/12d578afbb1f47e18e1f77eaaf88e3f0 is 50, key is test_row_0/A:col10/1731369327539/Put/seqid=0 2024-11-11T23:55:28,789 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/1099c313fa1c4d2b8f9ee5fed0f96cd3 is 50, key is test_row_0/B:col10/1731369327539/Put/seqid=0 2024-11-11T23:55:28,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742246_1422 (size=12139) 2024-11-11T23:55:28,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742247_1423 (size=12139) 2024-11-11T23:55:28,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T23:55:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:28,807 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:55:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9504d2f09e854802879449eef06c89c4 is 50, key is test_row_0/A:col10/1731369327556/Put/seqid=0 2024-11-11T23:55:28,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742248_1424 (size=12001) 2024-11-11T23:55:29,198 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/1099c313fa1c4d2b8f9ee5fed0f96cd3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1099c313fa1c4d2b8f9ee5fed0f96cd3 2024-11-11T23:55:29,201 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into 1099c313fa1c4d2b8f9ee5fed0f96cd3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:29,201 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:29,202 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=12, startTime=1731369328779; duration=0sec 2024-11-11T23:55:29,202 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:29,202 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:29,202 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:29,203 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:29,203 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:29,203 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:29,203 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a797629be6624eb497ba45b23a2ee4d5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/59c7ac7567e34fca991bc3001e030a81, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/bf567b4e6420493b84568da50ecabf70, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9900e905326a4307b842647bd16ba364] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=46.9 K 2024-11-11T23:55:29,203 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a797629be6624eb497ba45b23a2ee4d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731369325476 2024-11-11T23:55:29,203 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c7ac7567e34fca991bc3001e030a81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731369325568 2024-11-11T23:55:29,204 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting bf567b4e6420493b84568da50ecabf70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731369325691 2024-11-11T23:55:29,204 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9900e905326a4307b842647bd16ba364, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731369326377 2024-11-11T23:55:29,205 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/12d578afbb1f47e18e1f77eaaf88e3f0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/12d578afbb1f47e18e1f77eaaf88e3f0 2024-11-11T23:55:29,210 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into 12d578afbb1f47e18e1f77eaaf88e3f0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:29,210 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:29,210 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=12, startTime=1731369328779; duration=0sec 2024-11-11T23:55:29,210 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:29,210 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:29,216 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9504d2f09e854802879449eef06c89c4 2024-11-11T23:55:29,223 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:29,223 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/4a791bee11de4885a30cfdf258109d40 is 50, key is test_row_0/C:col10/1731369327539/Put/seqid=0 2024-11-11T23:55:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/742bcc0fd997415a98159a3a8de470b9 is 50, key is test_row_0/B:col10/1731369327556/Put/seqid=0 2024-11-11T23:55:29,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742249_1425 (size=12139) 2024-11-11T23:55:29,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742250_1426 (size=12001) 2024-11-11T23:55:29,633 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/4a791bee11de4885a30cfdf258109d40 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4a791bee11de4885a30cfdf258109d40 2024-11-11T23:55:29,636 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/742bcc0fd997415a98159a3a8de470b9 2024-11-11T23:55:29,637 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into 4a791bee11de4885a30cfdf258109d40(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:29,637 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:29,637 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=12, startTime=1731369328779; duration=0sec 2024-11-11T23:55:29,637 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:29,637 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/9e94a36114e641b29e1393cea9c4badb is 50, key is test_row_0/C:col10/1731369327556/Put/seqid=0 2024-11-11T23:55:29,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742251_1427 (size=12001) 2024-11-11T23:55:29,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T23:55:29,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369329790, max timestamp=1731369329804Num uniques 859; 2024-11-11T23:55:29,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369329790, max timestamp=1731369329804Num uniques 859; 2024-11-11T23:55:29,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:29,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:29,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369329790, max timestamp=1731369329804Num uniques 859; 2024-11-11T23:55:29,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:29,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369389830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369389836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369389837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369389838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369389839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369389944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369389944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369389954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369389954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369389965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,047 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/9e94a36114e641b29e1393cea9c4badb 2024-11-11T23:55:30,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9504d2f09e854802879449eef06c89c4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9504d2f09e854802879449eef06c89c4 2024-11-11T23:55:30,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9504d2f09e854802879449eef06c89c4, entries=150, sequenceid=103, filesize=11.7 K 2024-11-11T23:55:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/742bcc0fd997415a98159a3a8de470b9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/742bcc0fd997415a98159a3a8de470b9 2024-11-11T23:55:30,056 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/742bcc0fd997415a98159a3a8de470b9, entries=150, sequenceid=103, filesize=11.7 K 2024-11-11T23:55:30,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/9e94a36114e641b29e1393cea9c4badb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9e94a36114e641b29e1393cea9c4badb 2024-11-11T23:55:30,060 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9e94a36114e641b29e1393cea9c4badb, entries=150, sequenceid=103, filesize=11.7 K 2024-11-11T23:55:30,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6818f1e02fff6c00ff92806be77a8a11 in 1254ms, sequenceid=103, compaction requested=false 2024-11-11T23:55:30,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:30,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:30,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-11T23:55:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-11T23:55:30,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T23:55:30,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4750 sec 2024-11-11T23:55:30,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.4790 sec 2024-11-11T23:55:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:30,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T23:55:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:30,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:30,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9a9aebdd52a24c4ea71ecc734aee3096 is 50, key is test_row_0/A:col10/1731369329829/Put/seqid=0 2024-11-11T23:55:30,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742252_1428 (size=14541) 2024-11-11T23:55:30,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369390175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369390177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369390182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369390183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369390185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369390295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369390295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369390295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369390304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369390505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369390505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369390505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369390505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369390514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9a9aebdd52a24c4ea71ecc734aee3096 2024-11-11T23:55:30,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/60679a2bb5504f9796f4cefa4281bc7b is 50, key is test_row_0/B:col10/1731369329829/Put/seqid=0 2024-11-11T23:55:30,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742253_1429 (size=12151) 2024-11-11T23:55:30,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369390824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369390825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369390825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369390825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:30,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/60679a2bb5504f9796f4cefa4281bc7b 2024-11-11T23:55:30,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/405af0c5a66b486d83cd933fc53635da is 50, key is test_row_0/C:col10/1731369329829/Put/seqid=0 2024-11-11T23:55:31,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742254_1430 (size=12151) 2024-11-11T23:55:31,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369391024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:31,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369391335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:31,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369391335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:31,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369391345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369391346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:31,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/405af0c5a66b486d83cd933fc53635da 2024-11-11T23:55:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9a9aebdd52a24c4ea71ecc734aee3096 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9a9aebdd52a24c4ea71ecc734aee3096 2024-11-11T23:55:31,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9a9aebdd52a24c4ea71ecc734aee3096, entries=200, sequenceid=143, filesize=14.2 K 2024-11-11T23:55:31,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/60679a2bb5504f9796f4cefa4281bc7b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/60679a2bb5504f9796f4cefa4281bc7b 2024-11-11T23:55:31,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/60679a2bb5504f9796f4cefa4281bc7b, entries=150, sequenceid=143, filesize=11.9 K 2024-11-11T23:55:31,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/405af0c5a66b486d83cd933fc53635da as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/405af0c5a66b486d83cd933fc53635da 2024-11-11T23:55:31,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/405af0c5a66b486d83cd933fc53635da, entries=150, sequenceid=143, filesize=11.9 K 2024-11-11T23:55:31,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 6818f1e02fff6c00ff92806be77a8a11 in 1255ms, sequenceid=143, compaction requested=true 2024-11-11T23:55:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:31,426 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:31,426 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:31,426 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38681 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:31,426 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:31,426 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:31,426 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:31,427 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:31,427 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:31,427 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/12d578afbb1f47e18e1f77eaaf88e3f0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9504d2f09e854802879449eef06c89c4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9a9aebdd52a24c4ea71ecc734aee3096] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=37.8 K 2024-11-11T23:55:31,427 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1099c313fa1c4d2b8f9ee5fed0f96cd3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/742bcc0fd997415a98159a3a8de470b9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/60679a2bb5504f9796f4cefa4281bc7b] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=35.4 K 2024-11-11T23:55:31,427 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d578afbb1f47e18e1f77eaaf88e3f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731369326377 2024-11-11T23:55:31,427 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1099c313fa1c4d2b8f9ee5fed0f96cd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731369326377 2024-11-11T23:55:31,428 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9504d2f09e854802879449eef06c89c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731369327541 2024-11-11T23:55:31,428 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 742bcc0fd997415a98159a3a8de470b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731369327541 2024-11-11T23:55:31,428 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a9aebdd52a24c4ea71ecc734aee3096, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731369329822 2024-11-11T23:55:31,428 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 60679a2bb5504f9796f4cefa4281bc7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731369329822 2024-11-11T23:55:31,434 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:31,434 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/39697a48142a41eaa6c2d30818601ba7 is 50, key is test_row_0/A:col10/1731369329829/Put/seqid=0 2024-11-11T23:55:31,439 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:31,439 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/eeaced96ab184a2eafa22709b544206e is 50, key is test_row_0/B:col10/1731369329829/Put/seqid=0 2024-11-11T23:55:31,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742255_1431 (size=12391) 2024-11-11T23:55:31,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742256_1432 (size=12391) 2024-11-11T23:55:31,457 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/eeaced96ab184a2eafa22709b544206e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/eeaced96ab184a2eafa22709b544206e 2024-11-11T23:55:31,461 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into eeaced96ab184a2eafa22709b544206e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:31,461 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:31,461 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=13, startTime=1731369331426; duration=0sec 2024-11-11T23:55:31,461 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:31,461 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:31,461 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:31,463 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:31,463 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:31,463 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:31,463 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4a791bee11de4885a30cfdf258109d40, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9e94a36114e641b29e1393cea9c4badb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/405af0c5a66b486d83cd933fc53635da] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=35.4 K 2024-11-11T23:55:31,464 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a791bee11de4885a30cfdf258109d40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731369326377 2024-11-11T23:55:31,464 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e94a36114e641b29e1393cea9c4badb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731369327541 2024-11-11T23:55:31,464 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 405af0c5a66b486d83cd933fc53635da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731369329822 2024-11-11T23:55:31,472 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:31,473 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/aae8264be3c24955bb680c66ef655a33 is 50, key is test_row_0/C:col10/1731369329829/Put/seqid=0 2024-11-11T23:55:31,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742257_1433 (size=12391) 2024-11-11T23:55:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T23:55:31,745 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:31,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-11T23:55:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T23:55:31,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:31,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:31,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:31,851 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/39697a48142a41eaa6c2d30818601ba7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/39697a48142a41eaa6c2d30818601ba7 2024-11-11T23:55:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T23:55:31,855 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into 39697a48142a41eaa6c2d30818601ba7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:31,855 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:31,855 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=13, startTime=1731369331425; duration=0sec 2024-11-11T23:55:31,855 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:31,855 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:31,880 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/aae8264be3c24955bb680c66ef655a33 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/aae8264be3c24955bb680c66ef655a33 2024-11-11T23:55:31,884 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into aae8264be3c24955bb680c66ef655a33(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:31,884 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:31,884 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=13, startTime=1731369331426; duration=0sec 2024-11-11T23:55:31,884 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:31,884 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:31,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T23:55:31,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:31,899 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T23:55:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e0876e012eae451ea6397c9c90a521e1 is 50, key is test_row_0/A:col10/1731369330172/Put/seqid=0 2024-11-11T23:55:31,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742258_1434 (size=12151) 2024-11-11T23:55:31,906 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e0876e012eae451ea6397c9c90a521e1 2024-11-11T23:55:31,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/ac664ef038dd43ce976e5f32752959ea is 50, key is test_row_0/B:col10/1731369330172/Put/seqid=0 2024-11-11T23:55:31,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742259_1435 (size=12151) 2024-11-11T23:55:32,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T23:55:32,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369332121, max timestamp=1731369332140Num uniques 859; 2024-11-11T23:55:32,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:32,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369332121, max timestamp=1731369332140Num uniques 859; 2024-11-11T23:55:32,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:32,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369332121, max timestamp=1731369332140Num uniques 859; 2024-11-11T23:55:32,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:32,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369332234, max timestamp=1731369332273Num uniques 1018; 2024-11-11T23:55:32,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:55:32,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369332234, max timestamp=1731369332273Num uniques 1018; 2024-11-11T23:55:32,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:55:32,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369332234, max timestamp=1731369332273Num uniques 1018; 2024-11-11T23:55:32,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T23:55:32,321 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/ac664ef038dd43ce976e5f32752959ea 2024-11-11T23:55:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/93537cd0991f47a599d82f0272651fe6 is 50, key is test_row_0/C:col10/1731369330172/Put/seqid=0 2024-11-11T23:55:32,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742260_1436 (size=12151) 2024-11-11T23:55:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T23:55:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369392365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369392370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369392371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,384 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 6818f1e02fff6c00ff92806be77a8a11 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369392373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369332342, max timestamp=1731369332382Num uniques 1027; 2024-11-11T23:55:32,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:32,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369332342, max timestamp=1731369332382Num uniques 1027; 2024-11-11T23:55:32,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:32,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369332342, max timestamp=1731369332382Num uniques 1027; 2024-11-11T23:55:32,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:32,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731369332382, max timestamp=1731369332492Num uniques 1036; 2024-11-11T23:55:32,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731369332382, max timestamp=1731369332492Num uniques 1036; 2024-11-11T23:55:32,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731369332382, max timestamp=1731369332492Num uniques 1036; 2024-11-11T23:55:32,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T23:55:32,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T23:55:32,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T23:55:32,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369392535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1731369332527, max timestamp=1731369332542Num uniques 1045; 2024-11-11T23:55:32,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=11210, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T23:55:32,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1731369332527, max timestamp=1731369332542Num uniques 1045; 2024-11-11T23:55:32,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=11210, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T23:55:32,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1731369332527, max timestamp=1731369332542Num uniques 1045; 2024-11-11T23:55:32,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=11210, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T23:55:32,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369392591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369392592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369392593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369392655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369392704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369392714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369392714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,736 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/93537cd0991f47a599d82f0272651fe6 2024-11-11T23:55:32,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e0876e012eae451ea6397c9c90a521e1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e0876e012eae451ea6397c9c90a521e1 2024-11-11T23:55:32,741 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e0876e012eae451ea6397c9c90a521e1, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T23:55:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/ac664ef038dd43ce976e5f32752959ea as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/ac664ef038dd43ce976e5f32752959ea 2024-11-11T23:55:32,744 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/ac664ef038dd43ce976e5f32752959ea, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T23:55:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/93537cd0991f47a599d82f0272651fe6 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/93537cd0991f47a599d82f0272651fe6 2024-11-11T23:55:32,748 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/93537cd0991f47a599d82f0272651fe6, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T23:55:32,749 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 6818f1e02fff6c00ff92806be77a8a11 in 849ms, sequenceid=155, compaction requested=false 2024-11-11T23:55:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-11T23:55:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-11T23:55:32,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T23:55:32,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0030 sec 2024-11-11T23:55:32,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0070 sec 2024-11-11T23:55:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:32,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=147.60 KB heapSize=416.77 KB 2024-11-11T23:55:32,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:32,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:32,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:32,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:32,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:32,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T23:55:32,884 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:32,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:32,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/51136b15b66d4a9aa75180253b0df2c7 is 50, key is test_row_0/A:col10/1731369332592/Put/seqid=0 2024-11-11T23:55:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-11T23:55:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T23:55:32,887 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:32,888 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:32,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:32,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742261_1437 (size=14691) 2024-11-11T23:55:32,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/51136b15b66d4a9aa75180253b0df2c7 2024-11-11T23:55:32,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/c628e7ed14b54c908dec673289347a0c is 50, key is test_row_0/B:col10/1731369332592/Put/seqid=0 2024-11-11T23:55:32,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742262_1438 (size=12251) 2024-11-11T23:55:32,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369392914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369392924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369392924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369392935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T23:55:33,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T23:55:33,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:33,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:33,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:33,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:33,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:33,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369393035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T23:55:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:33,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T23:55:33,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369393234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369393234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369393245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369393254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/c628e7ed14b54c908dec673289347a0c 2024-11-11T23:55:33,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/048339648a6747c0b01a2b9f0078a66e is 50, key is test_row_0/C:col10/1731369332592/Put/seqid=0 2024-11-11T23:55:33,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742263_1439 (size=12251) 2024-11-11T23:55:33,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/048339648a6747c0b01a2b9f0078a66e 2024-11-11T23:55:33,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/51136b15b66d4a9aa75180253b0df2c7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/51136b15b66d4a9aa75180253b0df2c7 2024-11-11T23:55:33,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/51136b15b66d4a9aa75180253b0df2c7, entries=200, sequenceid=261, filesize=14.3 K 2024-11-11T23:55:33,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/c628e7ed14b54c908dec673289347a0c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/c628e7ed14b54c908dec673289347a0c 2024-11-11T23:55:33,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/c628e7ed14b54c908dec673289347a0c, entries=150, sequenceid=261, filesize=12.0 K 2024-11-11T23:55:33,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/048339648a6747c0b01a2b9f0078a66e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/048339648a6747c0b01a2b9f0078a66e 2024-11-11T23:55:33,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/048339648a6747c0b01a2b9f0078a66e, entries=150, sequenceid=261, filesize=12.0 K 2024-11-11T23:55:33,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~416.72 KB/426720, currentSize=33.54 KB/34350 for 6818f1e02fff6c00ff92806be77a8a11 in 447ms, sequenceid=261, compaction requested=true 2024-11-11T23:55:33,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:33,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:33,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:33,328 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:33,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:33,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:33,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:33,328 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:33,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:33,329 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:33,329 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:33,329 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:33,330 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/39697a48142a41eaa6c2d30818601ba7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e0876e012eae451ea6397c9c90a521e1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/51136b15b66d4a9aa75180253b0df2c7] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=38.3 K 2024-11-11T23:55:33,330 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39697a48142a41eaa6c2d30818601ba7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731369329822 2024-11-11T23:55:33,330 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0876e012eae451ea6397c9c90a521e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369330172 2024-11-11T23:55:33,330 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:33,330 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51136b15b66d4a9aa75180253b0df2c7, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731369332578 2024-11-11T23:55:33,330 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:33,330 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:33,330 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/eeaced96ab184a2eafa22709b544206e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/ac664ef038dd43ce976e5f32752959ea, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/c628e7ed14b54c908dec673289347a0c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=35.9 K 2024-11-11T23:55:33,331 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting eeaced96ab184a2eafa22709b544206e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731369329822 2024-11-11T23:55:33,331 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ac664ef038dd43ce976e5f32752959ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369330172 2024-11-11T23:55:33,331 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c628e7ed14b54c908dec673289347a0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731369332578 2024-11-11T23:55:33,339 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:33,340 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/b607d517f1284ebd9f8fa5949a8b3045 is 50, key is test_row_0/A:col10/1731369332592/Put/seqid=0 2024-11-11T23:55:33,344 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:33,344 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a2b4b598dfcd45f7a23eabb2af6f2d89 is 50, key is test_row_0/B:col10/1731369332592/Put/seqid=0 2024-11-11T23:55:33,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T23:55:33,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742264_1440 (size=12593) 2024-11-11T23:55:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:33,346 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T23:55:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:33,352 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/b607d517f1284ebd9f8fa5949a8b3045 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/b607d517f1284ebd9f8fa5949a8b3045 2024-11-11T23:55:33,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742265_1441 (size=12593) 2024-11-11T23:55:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/eb041d818b18470f9d0d0c4da235e9f9 is 50, key is test_row_0/A:col10/1731369332902/Put/seqid=0 2024-11-11T23:55:33,360 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a2b4b598dfcd45f7a23eabb2af6f2d89 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a2b4b598dfcd45f7a23eabb2af6f2d89 2024-11-11T23:55:33,360 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into b607d517f1284ebd9f8fa5949a8b3045(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:33,360 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:33,360 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=13, startTime=1731369333328; duration=0sec 2024-11-11T23:55:33,360 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:33,360 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:33,361 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:33,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742266_1442 (size=12301) 2024-11-11T23:55:33,366 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:33,366 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:33,367 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/eb041d818b18470f9d0d0c4da235e9f9 2024-11-11T23:55:33,366 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:33,367 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/aae8264be3c24955bb680c66ef655a33, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/93537cd0991f47a599d82f0272651fe6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/048339648a6747c0b01a2b9f0078a66e] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=35.9 K 2024-11-11T23:55:33,368 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into a2b4b598dfcd45f7a23eabb2af6f2d89(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:33,368 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:33,368 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=13, startTime=1731369333328; duration=0sec 2024-11-11T23:55:33,368 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:33,368 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:33,370 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting aae8264be3c24955bb680c66ef655a33, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731369329822 2024-11-11T23:55:33,371 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93537cd0991f47a599d82f0272651fe6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731369330172 2024-11-11T23:55:33,375 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 048339648a6747c0b01a2b9f0078a66e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731369332578 2024-11-11T23:55:33,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4cfccf6d96e64a3991551d657d95f2b7 is 50, key is test_row_0/B:col10/1731369332902/Put/seqid=0 2024-11-11T23:55:33,385 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#379 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:33,386 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/2e28ea27b7af4a4da9f4b83096d4e177 is 50, key is test_row_0/C:col10/1731369332592/Put/seqid=0 2024-11-11T23:55:33,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742267_1443 (size=12301) 2024-11-11T23:55:33,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742268_1444 (size=12593) 2024-11-11T23:55:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T23:55:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:33,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369333638, max timestamp=1731369333674Num uniques 859; 2024-11-11T23:55:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369333638, max timestamp=1731369333674Num uniques 859; 2024-11-11T23:55:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369333638, max timestamp=1731369333674Num uniques 859; 2024-11-11T23:55:33,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:33,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369393768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369333755, max timestamp=1731369333776Num uniques 1018; 2024-11-11T23:55:33,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T23:55:33,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369333755, max timestamp=1731369333776Num uniques 1018; 2024-11-11T23:55:33,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369333755, max timestamp=1731369333776Num uniques 1018; 2024-11-11T23:55:33,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T23:55:33,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T23:55:33,792 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4cfccf6d96e64a3991551d657d95f2b7 2024-11-11T23:55:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/3a9087fb5b43406eb4d61a1e575abd09 is 50, key is test_row_0/C:col10/1731369332902/Put/seqid=0 2024-11-11T23:55:33,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742269_1445 (size=12301) 2024-11-11T23:55:33,802 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/2e28ea27b7af4a4da9f4b83096d4e177 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/2e28ea27b7af4a4da9f4b83096d4e177 2024-11-11T23:55:33,805 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into 2e28ea27b7af4a4da9f4b83096d4e177(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:33,805 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:33,805 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=13, startTime=1731369333328; duration=0sec 2024-11-11T23:55:33,805 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:33,805 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:33,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731369333797, max timestamp=1731369333832Num uniques 1077; 2024-11-11T23:55:33,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T23:55:33,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731369333797, max timestamp=1731369333832Num uniques 1077; 2024-11-11T23:55:33,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731369333797, max timestamp=1731369333832Num uniques 1077; 2024-11-11T23:55:33,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T23:55:33,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T23:55:33,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369393884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369393885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369393886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:33,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369393895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369394004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369394005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369394015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T23:55:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369394105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/3a9087fb5b43406eb4d61a1e575abd09 2024-11-11T23:55:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/eb041d818b18470f9d0d0c4da235e9f9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/eb041d818b18470f9d0d0c4da235e9f9 2024-11-11T23:55:34,207 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/eb041d818b18470f9d0d0c4da235e9f9, entries=150, sequenceid=269, filesize=12.0 K 2024-11-11T23:55:34,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4cfccf6d96e64a3991551d657d95f2b7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4cfccf6d96e64a3991551d657d95f2b7 2024-11-11T23:55:34,210 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4cfccf6d96e64a3991551d657d95f2b7, entries=150, sequenceid=269, filesize=12.0 K 2024-11-11T23:55:34,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/3a9087fb5b43406eb4d61a1e575abd09 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/3a9087fb5b43406eb4d61a1e575abd09 2024-11-11T23:55:34,213 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/3a9087fb5b43406eb4d61a1e575abd09, entries=150, sequenceid=269, filesize=12.0 K 2024-11-11T23:55:34,214 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 6818f1e02fff6c00ff92806be77a8a11 in 868ms, sequenceid=269, compaction requested=false 2024-11-11T23:55:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-11T23:55:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-11T23:55:34,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T23:55:34,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3270 sec 2024-11-11T23:55:34,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.3310 sec 2024-11-11T23:55:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:34,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-11-11T23:55:34,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:34,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:34,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:34,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:34,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:34,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:34,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/4f99c52d016c4cf3a58b7bb7c76e32cd is 50, key is test_row_0/A:col10/1731369333865/Put/seqid=0 2024-11-11T23:55:34,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742270_1446 (size=17181) 2024-11-11T23:55:34,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369394240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369394248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369394253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369394364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369394364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369394374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369394395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,404 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 6818f1e02fff6c00ff92806be77a8a11 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369394424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369394574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369394574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369394594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/4f99c52d016c4cf3a58b7bb7c76e32cd 2024-11-11T23:55:34,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4f8b10ea6ca449bf9d1fd58a52372b04 is 50, key is test_row_0/B:col10/1731369333865/Put/seqid=0 2024-11-11T23:55:34,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742271_1447 (size=12301) 2024-11-11T23:55:34,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369394884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369394894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369394904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369394945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T23:55:35,035 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:35,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T23:55:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T23:55:35,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:35,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:35,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:35,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4f8b10ea6ca449bf9d1fd58a52372b04 2024-11-11T23:55:35,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/06120162ca9a44c3970fcaa189cae7e4 is 50, key is test_row_0/C:col10/1731369333865/Put/seqid=0 2024-11-11T23:55:35,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742272_1448 (size=12301) 2024-11-11T23:55:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T23:55:35,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T23:55:35,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:35,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:35,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:35,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:35,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:35,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T23:55:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:35,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T23:55:35,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369395404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:35,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369395415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:35,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369395415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:35,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/06120162ca9a44c3970fcaa189cae7e4 2024-11-11T23:55:35,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/4f99c52d016c4cf3a58b7bb7c76e32cd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/4f99c52d016c4cf3a58b7bb7c76e32cd 2024-11-11T23:55:35,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/4f99c52d016c4cf3a58b7bb7c76e32cd, entries=250, sequenceid=345, filesize=16.8 K 2024-11-11T23:55:35,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4f8b10ea6ca449bf9d1fd58a52372b04 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f8b10ea6ca449bf9d1fd58a52372b04 2024-11-11T23:55:35,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f8b10ea6ca449bf9d1fd58a52372b04, entries=150, sequenceid=345, filesize=12.0 K 2024-11-11T23:55:35,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/06120162ca9a44c3970fcaa189cae7e4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/06120162ca9a44c3970fcaa189cae7e4 2024-11-11T23:55:35,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/06120162ca9a44c3970fcaa189cae7e4, entries=150, sequenceid=345, filesize=12.0 K 2024-11-11T23:55:35,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~454.27 KB/465168, currentSize=20.13 KB/20610 for 6818f1e02fff6c00ff92806be77a8a11 in 1240ms, sequenceid=345, compaction requested=true 2024-11-11T23:55:35,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:35,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:35,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:35,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:35,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:35,477 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:35,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:35,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:35,477 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:35,478 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:35,478 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:35,478 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:35,478 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:35,479 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:35,479 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:35,479 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a2b4b598dfcd45f7a23eabb2af6f2d89, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4cfccf6d96e64a3991551d657d95f2b7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f8b10ea6ca449bf9d1fd58a52372b04] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.3 K 2024-11-11T23:55:35,479 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/b607d517f1284ebd9f8fa5949a8b3045, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/eb041d818b18470f9d0d0c4da235e9f9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/4f99c52d016c4cf3a58b7bb7c76e32cd] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=41.1 K 2024-11-11T23:55:35,479 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b4b598dfcd45f7a23eabb2af6f2d89, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731369332578 2024-11-11T23:55:35,479 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b607d517f1284ebd9f8fa5949a8b3045, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731369332578 2024-11-11T23:55:35,479 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cfccf6d96e64a3991551d657d95f2b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731369332885 2024-11-11T23:55:35,479 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb041d818b18470f9d0d0c4da235e9f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731369332885 2024-11-11T23:55:35,479 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8b10ea6ca449bf9d1fd58a52372b04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731369333865 2024-11-11T23:55:35,479 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f99c52d016c4cf3a58b7bb7c76e32cd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731369333865 2024-11-11T23:55:35,487 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:35,488 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/c318d78c53a84089966d2359f6a98d19 is 50, key is test_row_0/A:col10/1731369333865/Put/seqid=0 2024-11-11T23:55:35,489 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:35,490 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/1bc06a60d1f04cd4b9fa29ff981c8376 is 50, key is test_row_0/B:col10/1731369333865/Put/seqid=0 2024-11-11T23:55:35,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T23:55:35,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:35,495 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-11T23:55:35,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:35,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:35,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:35,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:35,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:35,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:35,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742273_1449 (size=12745) 2024-11-11T23:55:35,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/55bfc38234324786a619af2218a2fa55 is 50, key is test_row_0/A:col10/1731369334252/Put/seqid=0 2024-11-11T23:55:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742274_1450 (size=12745) 2024-11-11T23:55:35,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742275_1451 (size=9857) 2024-11-11T23:55:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T23:55:35,910 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/c318d78c53a84089966d2359f6a98d19 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/c318d78c53a84089966d2359f6a98d19 2024-11-11T23:55:35,916 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into c318d78c53a84089966d2359f6a98d19(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:35,916 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:35,916 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=13, startTime=1731369335477; duration=0sec 2024-11-11T23:55:35,916 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:35,916 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:35,916 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:35,917 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:35,917 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:35,917 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:35,917 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/2e28ea27b7af4a4da9f4b83096d4e177, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/3a9087fb5b43406eb4d61a1e575abd09, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/06120162ca9a44c3970fcaa189cae7e4] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.3 K 2024-11-11T23:55:35,917 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e28ea27b7af4a4da9f4b83096d4e177, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731369332578 2024-11-11T23:55:35,918 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a9087fb5b43406eb4d61a1e575abd09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731369332885 2024-11-11T23:55:35,918 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06120162ca9a44c3970fcaa189cae7e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731369333865 2024-11-11T23:55:35,923 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/55bfc38234324786a619af2218a2fa55 2024-11-11T23:55:35,926 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/1bc06a60d1f04cd4b9fa29ff981c8376 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1bc06a60d1f04cd4b9fa29ff981c8376 2024-11-11T23:55:35,932 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into 1bc06a60d1f04cd4b9fa29ff981c8376(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:35,932 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:35,932 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=13, startTime=1731369335477; duration=0sec 2024-11-11T23:55:35,932 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:35,932 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:35,933 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#387 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:35,933 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/4586b410c74e4b6ebef801e9f2214d6e is 50, key is test_row_0/C:col10/1731369333865/Put/seqid=0 2024-11-11T23:55:35,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4d6aa6e845134f819ffe597ba233e8c9 is 50, key is test_row_0/B:col10/1731369334252/Put/seqid=0 2024-11-11T23:55:35,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742277_1453 (size=12745) 2024-11-11T23:55:35,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742276_1452 (size=9857) 2024-11-11T23:55:35,948 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/4586b410c74e4b6ebef801e9f2214d6e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4586b410c74e4b6ebef801e9f2214d6e 2024-11-11T23:55:35,957 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into 4586b410c74e4b6ebef801e9f2214d6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:35,957 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:35,957 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=13, startTime=1731369335477; duration=0sec 2024-11-11T23:55:35,957 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:35,957 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:35,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369336014, max timestamp=1731369336039Num uniques 859; 2024-11-11T23:55:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369336014, max timestamp=1731369336039Num uniques 859; 2024-11-11T23:55:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369336014, max timestamp=1731369336039Num uniques 859; 2024-11-11T23:55:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:36,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369336099, max timestamp=1731369336137Num uniques 1068; 2024-11-11T23:55:36,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T23:55:36,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369336099, max timestamp=1731369336137Num uniques 1068; 2024-11-11T23:55:36,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T23:55:36,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369336099, max timestamp=1731369336137Num uniques 1068; 2024-11-11T23:55:36,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T23:55:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T23:55:36,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369336210, max timestamp=1731369336240Num uniques 1027; 2024-11-11T23:55:36,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369336210, max timestamp=1731369336240Num uniques 1027; 2024-11-11T23:55:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369336210, max timestamp=1731369336240Num uniques 1027; 2024-11-11T23:55:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:36,342 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4d6aa6e845134f819ffe597ba233e8c9 2024-11-11T23:55:36,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/89c427b05cfc4484a47688a4bbafddcb is 50, key is test_row_0/C:col10/1731369334252/Put/seqid=0 2024-11-11T23:55:36,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742278_1454 (size=9857) 2024-11-11T23:55:36,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731369336321, max timestamp=1731369336350Num uniques 1036; 2024-11-11T23:55:36,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T23:55:36,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731369336321, max timestamp=1731369336350Num uniques 1036; 2024-11-11T23:55:36,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T23:55:36,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731369336321, max timestamp=1731369336350Num uniques 1036; 2024-11-11T23:55:36,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T23:55:36,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731369336426, max timestamp=1731369336445Num uniques 1045; 2024-11-11T23:55:36,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731369336426, max timestamp=1731369336445Num uniques 1045; 2024-11-11T23:55:36,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-11T23:55:36,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-11T23:55:36,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731369336426, max timestamp=1731369336445Num uniques 1045; 2024-11-11T23:55:36,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-11T23:55:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369396487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369396490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731369336484, max timestamp=1731369336498Num uniques 1204; 2024-11-11T23:55:36,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=11630, suffix heap size=153184,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=254 2024-11-11T23:55:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731369336484, max timestamp=1731369336498Num uniques 1204; 2024-11-11T23:55:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=11630, suffix heap size=153184,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=254 2024-11-11T23:55:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731369336485, max timestamp=1731369336498Num uniques 1204; 2024-11-11T23:55:36,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-11T23:55:36,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731369336534, max timestamp=1731369336563Num uniques 1013; 2024-11-11T23:55:36,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-11-11T23:55:36,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731369336534, max timestamp=1731369336563Num uniques 1113; 2024-11-11T23:55:36,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-11T23:55:36,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731369336534, max timestamp=1731369336563Num uniques 1113; 2024-11-11T23:55:36,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-11T23:55:36,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1731369336587, max timestamp=1731369336617Num uniques 1122; 2024-11-11T23:55:36,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=14742, suffix heap size=144656,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=322 2024-11-11T23:55:36,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1731369336587, max timestamp=1731369336617Num uniques 1122; 2024-11-11T23:55:36,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1731369336587, max timestamp=1731369336617Num uniques 1122; 2024-11-11T23:55:36,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=14742, suffix heap size=144656,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=322 2024-11-11T23:55:36,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=14742, suffix heap size=144656,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=322 2024-11-11T23:55:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1731369336635, max timestamp=1731369336655Num uniques 1181; 2024-11-11T23:55:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1731369336635, max timestamp=1731369336655Num uniques 1181; 2024-11-11T23:55:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=15156, suffix heap size=150792,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=331 2024-11-11T23:55:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=15156, suffix heap size=150792,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=331 2024-11-11T23:55:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1731369336635, max timestamp=1731369336655Num uniques 1181; 2024-11-11T23:55:36,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=15156, suffix heap size=150792,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=331 2024-11-11T23:55:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369396691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731369336680, max timestamp=1731369336692Num uniques 1190; 2024-11-11T23:55:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=13280, suffix heap size=151728,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=290 2024-11-11T23:55:36,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731369336680, max timestamp=1731369336692Num uniques 1190; 2024-11-11T23:55:36,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=13280, suffix heap size=151728,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=290 2024-11-11T23:55:36,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731369336680, max timestamp=1731369336692Num uniques 1190; 2024-11-11T23:55:36,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=13280, suffix heap size=151728,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=290 2024-11-11T23:55:36,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731369336730, max timestamp=1731369336743Num uniques 1149; 2024-11-11T23:55:36,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-11T23:55:36,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731369336730, max timestamp=1731369336743Num uniques 1149; 2024-11-11T23:55:36,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-11T23:55:36,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731369336730, max timestamp=1731369336743Num uniques 1149; 2024-11-11T23:55:36,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-11T23:55:36,751 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/89c427b05cfc4484a47688a4bbafddcb 2024-11-11T23:55:36,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/55bfc38234324786a619af2218a2fa55 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/55bfc38234324786a619af2218a2fa55 2024-11-11T23:55:36,758 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/55bfc38234324786a619af2218a2fa55, entries=100, sequenceid=351, filesize=9.6 K 2024-11-11T23:55:36,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/4d6aa6e845134f819ffe597ba233e8c9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4d6aa6e845134f819ffe597ba233e8c9 2024-11-11T23:55:36,762 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4d6aa6e845134f819ffe597ba233e8c9, entries=100, sequenceid=351, filesize=9.6 K 2024-11-11T23:55:36,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/89c427b05cfc4484a47688a4bbafddcb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89c427b05cfc4484a47688a4bbafddcb 2024-11-11T23:55:36,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89c427b05cfc4484a47688a4bbafddcb, entries=100, sequenceid=351, filesize=9.6 K 2024-11-11T23:55:36,766 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=87.22 KB/89310 for 6818f1e02fff6c00ff92806be77a8a11 in 1271ms, sequenceid=351, compaction requested=false 2024-11-11T23:55:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T23:55:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T23:55:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:36,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=93.93 KB heapSize=252.33 KB 2024-11-11T23:55:36,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:36,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:36,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:36,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:36,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:36,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:36,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T23:55:36,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7300 sec 2024-11-11T23:55:36,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.7340 sec 2024-11-11T23:55:36,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e5a915dc086643fb97eb5b5367740339 is 50, key is test_row_0/A:col10/1731369336767/Put/seqid=0 2024-11-11T23:55:36,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742279_1455 (size=14741) 2024-11-11T23:55:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369396796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369396801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369396804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369396804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369396914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369396914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:36,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369396924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369397024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369397124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369397124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369397134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e5a915dc086643fb97eb5b5367740339 2024-11-11T23:55:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T23:55:37,184 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:37,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T23:55:37,186 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T23:55:37,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:37,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:37,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a4cb8a9ff41e4652a038107cb4e2306a is 50, key is test_row_0/B:col10/1731369336767/Put/seqid=0 2024-11-11T23:55:37,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742280_1456 (size=12301) 2024-11-11T23:55:37,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a4cb8a9ff41e4652a038107cb4e2306a 2024-11-11T23:55:37,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/11c235923435469d82629d82da5b51f3 is 50, key is test_row_0/C:col10/1731369336767/Put/seqid=0 2024-11-11T23:55:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742281_1457 (size=12301) 2024-11-11T23:55:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T23:55:37,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T23:55:37,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:37,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:37,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:37,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:37,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:37,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369397334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369397434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369397444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369397444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T23:55:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:37,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T23:55:37,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/11c235923435469d82629d82da5b51f3 2024-11-11T23:55:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e5a915dc086643fb97eb5b5367740339 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e5a915dc086643fb97eb5b5367740339 2024-11-11T23:55:37,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e5a915dc086643fb97eb5b5367740339, entries=200, sequenceid=554, filesize=14.4 K 2024-11-11T23:55:37,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a4cb8a9ff41e4652a038107cb4e2306a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a4cb8a9ff41e4652a038107cb4e2306a 2024-11-11T23:55:37,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a4cb8a9ff41e4652a038107cb4e2306a, entries=150, sequenceid=554, filesize=12.0 K 2024-11-11T23:55:37,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/11c235923435469d82629d82da5b51f3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/11c235923435469d82629d82da5b51f3 2024-11-11T23:55:37,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/11c235923435469d82629d82da5b51f3, entries=150, sequenceid=554, filesize=12.0 K 2024-11-11T23:55:37,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~252.28 KB/258336, currentSize=87.22 KB/89310 for 6818f1e02fff6c00ff92806be77a8a11 in 855ms, sequenceid=554, compaction requested=true 2024-11-11T23:55:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:37,623 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:37,623 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:37,624 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:37,624 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:37,624 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:37,624 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:37,624 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:37,624 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:37,624 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/c318d78c53a84089966d2359f6a98d19, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/55bfc38234324786a619af2218a2fa55, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e5a915dc086643fb97eb5b5367740339] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.5 K 2024-11-11T23:55:37,624 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1bc06a60d1f04cd4b9fa29ff981c8376, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4d6aa6e845134f819ffe597ba233e8c9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a4cb8a9ff41e4652a038107cb4e2306a] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=34.1 K 2024-11-11T23:55:37,624 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c318d78c53a84089966d2359f6a98d19, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731369333865 2024-11-11T23:55:37,624 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc06a60d1f04cd4b9fa29ff981c8376, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731369333865 2024-11-11T23:55:37,624 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d6aa6e845134f819ffe597ba233e8c9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731369334246 2024-11-11T23:55:37,624 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55bfc38234324786a619af2218a2fa55, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731369334246 2024-11-11T23:55:37,628 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a4cb8a9ff41e4652a038107cb4e2306a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731369336747 2024-11-11T23:55:37,628 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a915dc086643fb97eb5b5367740339, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731369336747 2024-11-11T23:55:37,634 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:37,634 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:37,634 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/faab52d28aff4edd9190a11b95490c7f is 50, key is test_row_0/B:col10/1731369336767/Put/seqid=0 2024-11-11T23:55:37,634 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/f7df8eefcc7c4d25924a7a8c69662af5 is 50, key is test_row_0/A:col10/1731369336767/Put/seqid=0 2024-11-11T23:55:37,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742283_1459 (size=12847) 2024-11-11T23:55:37,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742282_1458 (size=12847) 2024-11-11T23:55:37,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T23:55:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:37,645 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T23:55:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/0871cc0e713e49b78a5e6494a3e1c07c is 50, key is test_row_0/A:col10/1731369336799/Put/seqid=0 2024-11-11T23:55:37,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742284_1460 (size=12301) 2024-11-11T23:55:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T23:55:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:37,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369397945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369397954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369397965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369397965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,042 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/faab52d28aff4edd9190a11b95490c7f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/faab52d28aff4edd9190a11b95490c7f 2024-11-11T23:55:38,042 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/f7df8eefcc7c4d25924a7a8c69662af5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/f7df8eefcc7c4d25924a7a8c69662af5 2024-11-11T23:55:38,047 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into f7df8eefcc7c4d25924a7a8c69662af5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:38,047 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into faab52d28aff4edd9190a11b95490c7f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:38,047 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:38,047 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:38,047 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=13, startTime=1731369337623; duration=0sec 2024-11-11T23:55:38,047 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=13, startTime=1731369337623; duration=0sec 2024-11-11T23:55:38,048 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:38,048 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:38,048 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:38,048 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:38,048 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:38,048 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:38,048 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:38,048 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:38,049 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4586b410c74e4b6ebef801e9f2214d6e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89c427b05cfc4484a47688a4bbafddcb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/11c235923435469d82629d82da5b51f3] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=34.1 K 2024-11-11T23:55:38,049 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4586b410c74e4b6ebef801e9f2214d6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731369333865 2024-11-11T23:55:38,050 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c427b05cfc4484a47688a4bbafddcb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731369334246 2024-11-11T23:55:38,050 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11c235923435469d82629d82da5b51f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731369336747 2024-11-11T23:55:38,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/0871cc0e713e49b78a5e6494a3e1c07c 2024-11-11T23:55:38,056 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:38,057 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/b5e6da1c754c42cf92b6b4381c0aff8d is 50, key is test_row_0/C:col10/1731369336767/Put/seqid=0 2024-11-11T23:55:38,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/1f913f300cfd4f40a0bfce4ea3cce8bb is 50, key is test_row_0/B:col10/1731369336799/Put/seqid=0 2024-11-11T23:55:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369398065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742286_1462 (size=12301) 2024-11-11T23:55:38,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742285_1461 (size=12847) 2024-11-11T23:55:38,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/1f913f300cfd4f40a0bfce4ea3cce8bb 2024-11-11T23:55:38,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/10598733fa41430b87123d0666e59904 is 50, key is test_row_0/C:col10/1731369336799/Put/seqid=0 2024-11-11T23:55:38,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369398065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742287_1463 (size=12301) 2024-11-11T23:55:38,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369398274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369398285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T23:55:38,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33082 deadline: 1731369398444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,448 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 6818f1e02fff6c00ff92806be77a8a11 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,472 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/b5e6da1c754c42cf92b6b4381c0aff8d as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b5e6da1c754c42cf92b6b4381c0aff8d 2024-11-11T23:55:38,476 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into b5e6da1c754c42cf92b6b4381c0aff8d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:38,476 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:38,476 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=13, startTime=1731369337623; duration=0sec 2024-11-11T23:55:38,476 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:38,476 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:38,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/10598733fa41430b87123d0666e59904 2024-11-11T23:55:38,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/0871cc0e713e49b78a5e6494a3e1c07c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/0871cc0e713e49b78a5e6494a3e1c07c 2024-11-11T23:55:38,486 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/0871cc0e713e49b78a5e6494a3e1c07c, entries=150, sequenceid=570, filesize=12.0 K 2024-11-11T23:55:38,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/1f913f300cfd4f40a0bfce4ea3cce8bb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1f913f300cfd4f40a0bfce4ea3cce8bb 2024-11-11T23:55:38,489 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1f913f300cfd4f40a0bfce4ea3cce8bb, entries=150, sequenceid=570, filesize=12.0 K 2024-11-11T23:55:38,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/10598733fa41430b87123d0666e59904 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/10598733fa41430b87123d0666e59904 2024-11-11T23:55:38,496 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/10598733fa41430b87123d0666e59904, entries=150, sequenceid=570, filesize=12.0 K 2024-11-11T23:55:38,497 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6818f1e02fff6c00ff92806be77a8a11 in 852ms, sequenceid=570, compaction requested=false 2024-11-11T23:55:38,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:38,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:38,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T23:55:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T23:55:38,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T23:55:38,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3110 sec 2024-11-11T23:55:38,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3150 sec 2024-11-11T23:55:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:38,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T23:55:38,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:38,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/71bd54fdde2b4e0091f0e992fae5e972 is 50, key is test_row_0/A:col10/1731369338589/Put/seqid=0 2024-11-11T23:55:38,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742288_1464 (size=14741) 2024-11-11T23:55:38,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369398639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369398640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369398754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369398754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369398964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369398964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369398974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,983 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 6818f1e02fff6c00ff92806be77a8a11 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369398984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:38,992 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 6818f1e02fff6c00ff92806be77a8a11 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:38,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/71bd54fdde2b4e0091f0e992fae5e972 2024-11-11T23:55:39,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a9a2ef9dd33b4c1699a3599cc3a8ea79 is 50, key is test_row_0/B:col10/1731369338589/Put/seqid=0 2024-11-11T23:55:39,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742289_1465 (size=12301) 2024-11-11T23:55:39,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369399274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:39,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369399285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T23:55:39,345 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:39,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T23:55:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T23:55:39,347 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:39,347 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:39,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:39,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a9a2ef9dd33b4c1699a3599cc3a8ea79 2024-11-11T23:55:39,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/d0142cb92aa74143a15ed391dcd137a7 is 50, key is test_row_0/C:col10/1731369338589/Put/seqid=0 2024-11-11T23:55:39,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742290_1466 (size=12301) 2024-11-11T23:55:39,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/d0142cb92aa74143a15ed391dcd137a7 2024-11-11T23:55:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/71bd54fdde2b4e0091f0e992fae5e972 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/71bd54fdde2b4e0091f0e992fae5e972 2024-11-11T23:55:39,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/71bd54fdde2b4e0091f0e992fae5e972, entries=200, sequenceid=590, filesize=14.4 K 2024-11-11T23:55:39,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a9a2ef9dd33b4c1699a3599cc3a8ea79 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a9a2ef9dd33b4c1699a3599cc3a8ea79 2024-11-11T23:55:39,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a9a2ef9dd33b4c1699a3599cc3a8ea79, entries=150, sequenceid=590, filesize=12.0 K 2024-11-11T23:55:39,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/d0142cb92aa74143a15ed391dcd137a7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d0142cb92aa74143a15ed391dcd137a7 2024-11-11T23:55:39,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d0142cb92aa74143a15ed391dcd137a7, entries=150, sequenceid=590, filesize=12.0 K 2024-11-11T23:55:39,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6818f1e02fff6c00ff92806be77a8a11 in 846ms, sequenceid=590, compaction requested=true 2024-11-11T23:55:39,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:39,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:39,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:39,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:39,437 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:39,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:39,437 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:39,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:39,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:39,438 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:39,438 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:39,438 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:39,438 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:39,438 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:39,438 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:39,438 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/faab52d28aff4edd9190a11b95490c7f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1f913f300cfd4f40a0bfce4ea3cce8bb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a9a2ef9dd33b4c1699a3599cc3a8ea79] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.6 K 2024-11-11T23:55:39,438 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/f7df8eefcc7c4d25924a7a8c69662af5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/0871cc0e713e49b78a5e6494a3e1c07c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/71bd54fdde2b4e0091f0e992fae5e972] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=39.0 K 2024-11-11T23:55:39,438 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting faab52d28aff4edd9190a11b95490c7f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731369336747 2024-11-11T23:55:39,438 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7df8eefcc7c4d25924a7a8c69662af5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731369336747 2024-11-11T23:55:39,438 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f913f300cfd4f40a0bfce4ea3cce8bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731369336794 2024-11-11T23:55:39,438 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0871cc0e713e49b78a5e6494a3e1c07c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731369336794 2024-11-11T23:55:39,439 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71bd54fdde2b4e0091f0e992fae5e972, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731369337896 2024-11-11T23:55:39,439 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a2ef9dd33b4c1699a3599cc3a8ea79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731369337940 2024-11-11T23:55:39,449 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:39,449 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/d46610944780455daff6999ffd8bcd0f is 50, key is test_row_0/B:col10/1731369338589/Put/seqid=0 2024-11-11T23:55:39,452 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:39,452 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/7455374b475f4b7297ddd92a5cd8aeb4 is 50, key is test_row_0/A:col10/1731369338589/Put/seqid=0 2024-11-11T23:55:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T23:55:39,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742291_1467 (size=12949) 2024-11-11T23:55:39,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742292_1468 (size=12949) 2024-11-11T23:55:39,461 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/7455374b475f4b7297ddd92a5cd8aeb4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/7455374b475f4b7297ddd92a5cd8aeb4 2024-11-11T23:55:39,464 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into 7455374b475f4b7297ddd92a5cd8aeb4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:39,465 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:39,465 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=13, startTime=1731369339437; duration=0sec 2024-11-11T23:55:39,465 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:39,465 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:39,465 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:39,465 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:39,465 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:39,465 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:39,466 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b5e6da1c754c42cf92b6b4381c0aff8d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/10598733fa41430b87123d0666e59904, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d0142cb92aa74143a15ed391dcd137a7] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.6 K 2024-11-11T23:55:39,466 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5e6da1c754c42cf92b6b4381c0aff8d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731369336747 2024-11-11T23:55:39,466 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10598733fa41430b87123d0666e59904, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731369336794 2024-11-11T23:55:39,466 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0142cb92aa74143a15ed391dcd137a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731369337940 2024-11-11T23:55:39,472 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:39,473 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/e5489b813fca4f7bb2652d49fbecac30 is 50, key is test_row_0/C:col10/1731369338589/Put/seqid=0 2024-11-11T23:55:39,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742293_1469 (size=12949) 2024-11-11T23:55:39,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T23:55:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:39,500 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T23:55:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:39,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9f297c0145ef4eb29cfbe69ec90447c3 is 50, key is test_row_0/A:col10/1731369338638/Put/seqid=0 2024-11-11T23:55:39,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742294_1470 (size=12301) 2024-11-11T23:55:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T23:55:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:39,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:39,858 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/d46610944780455daff6999ffd8bcd0f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d46610944780455daff6999ffd8bcd0f 2024-11-11T23:55:39,862 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into d46610944780455daff6999ffd8bcd0f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:39,862 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:39,862 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=13, startTime=1731369339437; duration=0sec 2024-11-11T23:55:39,862 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:39,862 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369399860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:39,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369399865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:39,880 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/e5489b813fca4f7bb2652d49fbecac30 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/e5489b813fca4f7bb2652d49fbecac30 2024-11-11T23:55:39,883 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into e5489b813fca4f7bb2652d49fbecac30(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:39,883 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:39,883 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=13, startTime=1731369339437; duration=0sec 2024-11-11T23:55:39,883 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:39,883 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:39,908 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9f297c0145ef4eb29cfbe69ec90447c3 2024-11-11T23:55:39,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/000497c372af447298a23a7706eb7634 is 50, key is test_row_0/B:col10/1731369338638/Put/seqid=0 2024-11-11T23:55:39,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742295_1471 (size=12301) 2024-11-11T23:55:39,918 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/000497c372af447298a23a7706eb7634 2024-11-11T23:55:39,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/b827db4fb0c54859a430f38c886a3a56 is 50, key is test_row_0/C:col10/1731369338638/Put/seqid=0 2024-11-11T23:55:39,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742296_1472 (size=12301) 2024-11-11T23:55:39,927 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/b827db4fb0c54859a430f38c886a3a56 2024-11-11T23:55:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9f297c0145ef4eb29cfbe69ec90447c3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9f297c0145ef4eb29cfbe69ec90447c3 2024-11-11T23:55:39,932 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9f297c0145ef4eb29cfbe69ec90447c3, entries=150, sequenceid=606, filesize=12.0 K 2024-11-11T23:55:39,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/000497c372af447298a23a7706eb7634 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/000497c372af447298a23a7706eb7634 2024-11-11T23:55:39,935 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/000497c372af447298a23a7706eb7634, entries=150, sequenceid=606, filesize=12.0 K 2024-11-11T23:55:39,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/b827db4fb0c54859a430f38c886a3a56 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b827db4fb0c54859a430f38c886a3a56 2024-11-11T23:55:39,938 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b827db4fb0c54859a430f38c886a3a56, entries=150, sequenceid=606, filesize=12.0 K 2024-11-11T23:55:39,939 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6818f1e02fff6c00ff92806be77a8a11 in 438ms, sequenceid=606, compaction requested=false 2024-11-11T23:55:39,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:39,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:39,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T23:55:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T23:55:39,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T23:55:39,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 592 msec 2024-11-11T23:55:39,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 596 msec 2024-11-11T23:55:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T23:55:39,975 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:39,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T23:55:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T23:55:39,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:39,977 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:39,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:39,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T23:55:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:39,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9592fc03704b457599ba76f66b3c2808 is 50, key is test_row_0/A:col10/1731369339976/Put/seqid=0 2024-11-11T23:55:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742297_1473 (size=14741) 2024-11-11T23:55:40,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369400015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:40,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369400019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T23:55:40,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:40,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:40,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369400134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369400134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:40,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:40,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:40,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T23:55:40,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369400355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:40,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369400355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:40,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:40,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:40,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9592fc03704b457599ba76f66b3c2808 2024-11-11T23:55:40,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/d4188734fde64a92a24bcca1f1f7a8df is 50, key is test_row_0/B:col10/1731369339976/Put/seqid=0 2024-11-11T23:55:40,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742298_1474 (size=12301) 2024-11-11T23:55:40,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:40,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:40,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:40,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:40,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T23:55:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369400674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:40,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369400674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:40,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:40,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:40,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/d4188734fde64a92a24bcca1f1f7a8df 2024-11-11T23:55:40,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/89519ce01ff7407496d235b0b501ca32 is 50, key is test_row_0/C:col10/1731369339976/Put/seqid=0 2024-11-11T23:55:40,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742299_1475 (size=12301) 2024-11-11T23:55:40,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:40,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:40,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:40,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:41,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369400995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:41,002 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 6818f1e02fff6c00ff92806be77a8a11 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:41,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369401014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:41,017 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 6818f1e02fff6c00ff92806be77a8a11 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:41,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:41,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:41,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T23:55:41,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:41,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:41,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:41,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369401195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:41,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369401195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:41,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/89519ce01ff7407496d235b0b501ca32 2024-11-11T23:55:41,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/9592fc03704b457599ba76f66b3c2808 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9592fc03704b457599ba76f66b3c2808 2024-11-11T23:55:41,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9592fc03704b457599ba76f66b3c2808, entries=200, sequenceid=626, filesize=14.4 K 2024-11-11T23:55:41,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/d4188734fde64a92a24bcca1f1f7a8df as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d4188734fde64a92a24bcca1f1f7a8df 2024-11-11T23:55:41,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d4188734fde64a92a24bcca1f1f7a8df, entries=150, sequenceid=626, filesize=12.0 K 2024-11-11T23:55:41,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/89519ce01ff7407496d235b0b501ca32 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89519ce01ff7407496d235b0b501ca32 2024-11-11T23:55:41,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89519ce01ff7407496d235b0b501ca32, entries=150, sequenceid=626, filesize=12.0 K 2024-11-11T23:55:41,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6818f1e02fff6c00ff92806be77a8a11 in 1239ms, sequenceid=626, compaction requested=true 2024-11-11T23:55:41,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:41,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:41,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:41,221 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:41,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:41,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:41,221 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:41,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:41,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:41,221 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:41,221 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:41,221 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:41,221 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:41,221 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:41,222 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:41,222 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d46610944780455daff6999ffd8bcd0f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/000497c372af447298a23a7706eb7634, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d4188734fde64a92a24bcca1f1f7a8df] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.7 K 2024-11-11T23:55:41,222 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/7455374b475f4b7297ddd92a5cd8aeb4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9f297c0145ef4eb29cfbe69ec90447c3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9592fc03704b457599ba76f66b3c2808] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=39.1 K 2024-11-11T23:55:41,222 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting d46610944780455daff6999ffd8bcd0f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731369337940 2024-11-11T23:55:41,222 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7455374b475f4b7297ddd92a5cd8aeb4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731369337940 2024-11-11T23:55:41,222 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 000497c372af447298a23a7706eb7634, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731369338628 2024-11-11T23:55:41,222 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f297c0145ef4eb29cfbe69ec90447c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731369338628 2024-11-11T23:55:41,222 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting d4188734fde64a92a24bcca1f1f7a8df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731369339849 2024-11-11T23:55:41,222 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9592fc03704b457599ba76f66b3c2808, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731369339849 2024-11-11T23:55:41,230 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:41,230 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:41,230 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/df5cf20a3e0e42f4806644024cca3f1a is 50, key is test_row_0/B:col10/1731369339976/Put/seqid=0 2024-11-11T23:55:41,230 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/761cfe6aa2b8401daabea521cea0205c is 50, key is test_row_0/A:col10/1731369339976/Put/seqid=0 2024-11-11T23:55:41,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742301_1477 (size=13051) 2024-11-11T23:55:41,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742300_1476 (size=13051) 2024-11-11T23:55:41,244 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/761cfe6aa2b8401daabea521cea0205c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/761cfe6aa2b8401daabea521cea0205c 2024-11-11T23:55:41,247 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into 761cfe6aa2b8401daabea521cea0205c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:41,247 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:41,247 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=13, startTime=1731369341221; duration=0sec 2024-11-11T23:55:41,247 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:41,247 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:41,247 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:41,248 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:41,248 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:41,248 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:41,248 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/e5489b813fca4f7bb2652d49fbecac30, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b827db4fb0c54859a430f38c886a3a56, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89519ce01ff7407496d235b0b501ca32] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.7 K 2024-11-11T23:55:41,249 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5489b813fca4f7bb2652d49fbecac30, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731369337940 2024-11-11T23:55:41,249 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b827db4fb0c54859a430f38c886a3a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731369338628 2024-11-11T23:55:41,249 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89519ce01ff7407496d235b0b501ca32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731369339849 2024-11-11T23:55:41,260 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:41,260 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/25ee6d63bec54ef780fd255c020cb4e5 is 50, key is test_row_0/C:col10/1731369339976/Put/seqid=0 2024-11-11T23:55:41,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742302_1478 (size=13051) 2024-11-11T23:55:41,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T23:55:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:41,349 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T23:55:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:41,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/53a35d60870b47c88153d71ce92240e5 is 50, key is test_row_0/A:col10/1731369339996/Put/seqid=0 2024-11-11T23:55:41,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742303_1479 (size=12301) 2024-11-11T23:55:41,637 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/df5cf20a3e0e42f4806644024cca3f1a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/df5cf20a3e0e42f4806644024cca3f1a 2024-11-11T23:55:41,641 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into df5cf20a3e0e42f4806644024cca3f1a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:41,641 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:41,641 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=13, startTime=1731369341221; duration=0sec 2024-11-11T23:55:41,641 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:41,641 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:41,671 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/25ee6d63bec54ef780fd255c020cb4e5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/25ee6d63bec54ef780fd255c020cb4e5 2024-11-11T23:55:41,675 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into 25ee6d63bec54ef780fd255c020cb4e5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:41,675 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:41,675 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=13, startTime=1731369341221; duration=0sec 2024-11-11T23:55:41,676 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:41,676 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:41,756 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/53a35d60870b47c88153d71ce92240e5 2024-11-11T23:55:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/f3af9b143b4e45e0812656e57d22d5e6 is 50, key is test_row_0/B:col10/1731369339996/Put/seqid=0 2024-11-11T23:55:41,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742304_1480 (size=12301) 2024-11-11T23:55:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T23:55:42,170 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/f3af9b143b4e45e0812656e57d22d5e6 2024-11-11T23:55:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/4ae6a8a8e4f04f1480e455d19e186c4e is 50, key is test_row_0/C:col10/1731369339996/Put/seqid=0 2024-11-11T23:55:42,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742305_1481 (size=12301) 2024-11-11T23:55:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:42,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:42,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:42,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369402254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369402258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:42,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369402365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369402374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:42,580 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/4ae6a8a8e4f04f1480e455d19e186c4e 2024-11-11T23:55:42,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369402575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:42,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/53a35d60870b47c88153d71ce92240e5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/53a35d60870b47c88153d71ce92240e5 2024-11-11T23:55:42,586 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/53a35d60870b47c88153d71ce92240e5, entries=150, sequenceid=641, filesize=12.0 K 2024-11-11T23:55:42,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/f3af9b143b4e45e0812656e57d22d5e6 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f3af9b143b4e45e0812656e57d22d5e6 2024-11-11T23:55:42,589 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f3af9b143b4e45e0812656e57d22d5e6, entries=150, sequenceid=641, filesize=12.0 K 2024-11-11T23:55:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/4ae6a8a8e4f04f1480e455d19e186c4e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4ae6a8a8e4f04f1480e455d19e186c4e 2024-11-11T23:55:42,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369402584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:42,593 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4ae6a8a8e4f04f1480e455d19e186c4e, entries=150, sequenceid=641, filesize=12.0 K 2024-11-11T23:55:42,593 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6818f1e02fff6c00ff92806be77a8a11 in 1244ms, sequenceid=641, compaction requested=false 2024-11-11T23:55:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T23:55:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T23:55:42,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T23:55:42,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6170 sec 2024-11-11T23:55:42,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6210 sec 2024-11-11T23:55:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:42,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T23:55:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:42,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/df9dbc177ded482f8e6409c6bed33c12 is 50, key is test_row_0/A:col10/1731369342896/Put/seqid=0 2024-11-11T23:55:42,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742306_1482 (size=14741) 2024-11-11T23:55:42,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/df9dbc177ded482f8e6409c6bed33c12 2024-11-11T23:55:42,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/311921c24d88428f99a592543357a6a5 is 50, key is test_row_0/B:col10/1731369342896/Put/seqid=0 2024-11-11T23:55:42,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742307_1483 (size=12301) 2024-11-11T23:55:42,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369402935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:42,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369402937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:42,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:42,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:43,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369403054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:43,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369403054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369403265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:43,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369403274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:43,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/311921c24d88428f99a592543357a6a5 2024-11-11T23:55:43,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/634e3b4194344988bd25524669db2738 is 50, key is test_row_0/C:col10/1731369342896/Put/seqid=0 2024-11-11T23:55:43,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742308_1484 (size=12301) 2024-11-11T23:55:43,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369403584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:43,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369403595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:43,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/634e3b4194344988bd25524669db2738 2024-11-11T23:55:43,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/df9dbc177ded482f8e6409c6bed33c12 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/df9dbc177ded482f8e6409c6bed33c12 2024-11-11T23:55:43,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/df9dbc177ded482f8e6409c6bed33c12, entries=200, sequenceid=662, filesize=14.4 K 2024-11-11T23:55:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/311921c24d88428f99a592543357a6a5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/311921c24d88428f99a592543357a6a5 2024-11-11T23:55:43,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/311921c24d88428f99a592543357a6a5, entries=150, sequenceid=662, filesize=12.0 K 2024-11-11T23:55:43,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/634e3b4194344988bd25524669db2738 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/634e3b4194344988bd25524669db2738 2024-11-11T23:55:43,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/634e3b4194344988bd25524669db2738, entries=150, sequenceid=662, filesize=12.0 K 2024-11-11T23:55:43,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6818f1e02fff6c00ff92806be77a8a11 in 838ms, sequenceid=662, compaction requested=true 2024-11-11T23:55:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:43,735 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:43,735 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:43,736 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:43,736 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:43,736 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:43,736 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:43,736 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:43,736 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:43,736 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/761cfe6aa2b8401daabea521cea0205c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/53a35d60870b47c88153d71ce92240e5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/df9dbc177ded482f8e6409c6bed33c12] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=39.2 K 2024-11-11T23:55:43,736 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/df5cf20a3e0e42f4806644024cca3f1a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f3af9b143b4e45e0812656e57d22d5e6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/311921c24d88428f99a592543357a6a5] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.8 K 2024-11-11T23:55:43,736 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761cfe6aa2b8401daabea521cea0205c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731369339849 2024-11-11T23:55:43,736 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting df5cf20a3e0e42f4806644024cca3f1a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731369339849 2024-11-11T23:55:43,737 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f3af9b143b4e45e0812656e57d22d5e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731369339996 2024-11-11T23:55:43,737 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53a35d60870b47c88153d71ce92240e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731369339996 2024-11-11T23:55:43,737 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 311921c24d88428f99a592543357a6a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731369342246 2024-11-11T23:55:43,737 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting df9dbc177ded482f8e6409c6bed33c12, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731369342246 2024-11-11T23:55:43,742 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:43,742 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:43,743 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/abe39a39fb0b42728657ee1e7080bb96 is 50, key is test_row_0/B:col10/1731369342896/Put/seqid=0 2024-11-11T23:55:43,743 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/93d2b9ca351a454e8a7dc60b75627521 is 50, key is test_row_0/A:col10/1731369342896/Put/seqid=0 2024-11-11T23:55:43,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742309_1485 (size=13153) 2024-11-11T23:55:43,753 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/93d2b9ca351a454e8a7dc60b75627521 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/93d2b9ca351a454e8a7dc60b75627521 2024-11-11T23:55:43,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742310_1486 (size=13153) 2024-11-11T23:55:43,756 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into 93d2b9ca351a454e8a7dc60b75627521(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:43,757 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:43,757 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=13, startTime=1731369343735; duration=0sec 2024-11-11T23:55:43,757 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:43,757 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:43,757 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:43,757 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:43,758 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:43,758 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:43,758 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/25ee6d63bec54ef780fd255c020cb4e5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4ae6a8a8e4f04f1480e455d19e186c4e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/634e3b4194344988bd25524669db2738] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.8 K 2024-11-11T23:55:43,758 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ee6d63bec54ef780fd255c020cb4e5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731369339849 2024-11-11T23:55:43,758 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae6a8a8e4f04f1480e455d19e186c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731369339996 2024-11-11T23:55:43,758 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 634e3b4194344988bd25524669db2738, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731369342246 2024-11-11T23:55:43,763 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:43,763 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/a3499ed4ad8c46008d74eb0aa94b31ef is 50, key is test_row_0/C:col10/1731369342896/Put/seqid=0 2024-11-11T23:55:43,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742311_1487 (size=13153) 2024-11-11T23:55:43,769 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/a3499ed4ad8c46008d74eb0aa94b31ef as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a3499ed4ad8c46008d74eb0aa94b31ef 2024-11-11T23:55:43,772 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into a3499ed4ad8c46008d74eb0aa94b31ef(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:43,772 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:43,772 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=13, startTime=1731369343735; duration=0sec 2024-11-11T23:55:43,772 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:43,772 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:44,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T23:55:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:44,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/1f05c46944624f24abe7114e5dc15985 is 50, key is test_row_0/A:col10/1731369342916/Put/seqid=0 2024-11-11T23:55:44,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742312_1488 (size=14741) 2024-11-11T23:55:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T23:55:44,144 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:44,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T23:55:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T23:55:44,146 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:44,146 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:44,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:44,154 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/abe39a39fb0b42728657ee1e7080bb96 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/abe39a39fb0b42728657ee1e7080bb96 2024-11-11T23:55:44,158 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into abe39a39fb0b42728657ee1e7080bb96(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:44,158 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:44,158 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=13, startTime=1731369343735; duration=0sec 2024-11-11T23:55:44,158 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:44,158 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369404154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:44,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369404155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T23:55:44,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369404264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:44,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369404274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:44,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:44,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:44,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:44,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:44,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T23:55:44,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369404475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369404484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:44,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/1f05c46944624f24abe7114e5dc15985 2024-11-11T23:55:44,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a7c9223614c440c790e00f7a67a8dcdb is 50, key is test_row_0/B:col10/1731369342916/Put/seqid=0 2024-11-11T23:55:44,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742313_1489 (size=12301) 2024-11-11T23:55:44,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:44,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:44,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:44,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:44,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T23:55:44,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369404784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:44,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369404795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:44,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:44,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:44,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:44,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:44,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a7c9223614c440c790e00f7a67a8dcdb 2024-11-11T23:55:44,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/d52e8f2e7630448994244928fed1bd2f is 50, key is test_row_0/C:col10/1731369342916/Put/seqid=0 2024-11-11T23:55:44,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742314_1490 (size=12301) 2024-11-11T23:55:44,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/d52e8f2e7630448994244928fed1bd2f 2024-11-11T23:55:44,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/1f05c46944624f24abe7114e5dc15985 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1f05c46944624f24abe7114e5dc15985 2024-11-11T23:55:44,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1f05c46944624f24abe7114e5dc15985, entries=200, sequenceid=678, filesize=14.4 K 2024-11-11T23:55:44,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/a7c9223614c440c790e00f7a67a8dcdb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a7c9223614c440c790e00f7a67a8dcdb 2024-11-11T23:55:44,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a7c9223614c440c790e00f7a67a8dcdb, entries=150, sequenceid=678, filesize=12.0 K 2024-11-11T23:55:44,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/d52e8f2e7630448994244928fed1bd2f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d52e8f2e7630448994244928fed1bd2f 2024-11-11T23:55:44,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d52e8f2e7630448994244928fed1bd2f, entries=150, sequenceid=678, filesize=12.0 K 2024-11-11T23:55:44,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6818f1e02fff6c00ff92806be77a8a11 in 843ms, sequenceid=678, compaction requested=false 2024-11-11T23:55:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:45,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T23:55:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:45,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/162fe23cc0bc4130ade01428fcdf122c is 50, key is test_row_0/A:col10/1731369344154/Put/seqid=0 2024-11-11T23:55:45,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742315_1491 (size=14741) 2024-11-11T23:55:45,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:45,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:45,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369405067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369405070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369405185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369405185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T23:55:45,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33136 deadline: 1731369405304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33084 deadline: 1731369405314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:45,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:45,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369405394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369405404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/162fe23cc0bc4130ade01428fcdf122c 2024-11-11T23:55:45,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/099bb9d6dff7499f8cddbde8be135c4e is 50, key is test_row_0/B:col10/1731369344154/Put/seqid=0 2024-11-11T23:55:45,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742316_1492 (size=12301) 2024-11-11T23:55:45,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/099bb9d6dff7499f8cddbde8be135c4e 2024-11-11T23:55:45,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/17d7c6797cdc4b949ebd7bcee6c3e6de is 50, key is test_row_0/C:col10/1731369344154/Put/seqid=0 2024-11-11T23:55:45,450 INFO [Thread-1871 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1871. 2024-11-11T23:55:45,450 DEBUG [Thread-1871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:45,450 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,450 INFO [Thread-1865 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1865. 2024-11-11T23:55:45,450 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:45,450 DEBUG [Thread-1865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:45,450 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,450 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:45,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742317_1493 (size=12301) 2024-11-11T23:55:45,453 INFO [Thread-1867 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1867. 2024-11-11T23:55:45,453 DEBUG [Thread-1867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:45,453 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,453 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:45,461 INFO [Thread-1869 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1869. 2024-11-11T23:55:45,461 DEBUG [Thread-1869 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:45,461 DEBUG [Thread-1869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,461 DEBUG [Thread-1869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:45,462 INFO [Thread-1873 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1873. 2024-11-11T23:55:45,462 DEBUG [Thread-1873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:45,462 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,462 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:45,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:45,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:45,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:45,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731369405705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33156 deadline: 1731369405714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 , the old value is region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., hostname=b70d29849faf,42393,1731369201430, seqNum=2 because the exception is null or not the one we care about 2024-11-11T23:55:45,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:45,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:45,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:45,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/17d7c6797cdc4b949ebd7bcee6c3e6de 2024-11-11T23:55:45,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/162fe23cc0bc4130ade01428fcdf122c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/162fe23cc0bc4130ade01428fcdf122c 2024-11-11T23:55:45,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/162fe23cc0bc4130ade01428fcdf122c, entries=200, sequenceid=698, filesize=14.4 K 2024-11-11T23:55:45,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/099bb9d6dff7499f8cddbde8be135c4e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/099bb9d6dff7499f8cddbde8be135c4e 2024-11-11T23:55:45,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/099bb9d6dff7499f8cddbde8be135c4e, entries=150, sequenceid=698, filesize=12.0 K 2024-11-11T23:55:45,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/17d7c6797cdc4b949ebd7bcee6c3e6de as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/17d7c6797cdc4b949ebd7bcee6c3e6de 2024-11-11T23:55:45,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/17d7c6797cdc4b949ebd7bcee6c3e6de, entries=150, sequenceid=698, filesize=12.0 K 2024-11-11T23:55:45,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6818f1e02fff6c00ff92806be77a8a11 in 858ms, sequenceid=698, compaction requested=true 2024-11-11T23:55:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:45,878 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6818f1e02fff6c00ff92806be77a8a11:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:45,878 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:45,879 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:45,879 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:45,880 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/A is initiating minor compaction (all files) 2024-11-11T23:55:45,880 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/B is initiating minor compaction (all files) 2024-11-11T23:55:45,880 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/A in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,880 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/B in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,880 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/abe39a39fb0b42728657ee1e7080bb96, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a7c9223614c440c790e00f7a67a8dcdb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/099bb9d6dff7499f8cddbde8be135c4e] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.9 K 2024-11-11T23:55:45,880 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/93d2b9ca351a454e8a7dc60b75627521, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1f05c46944624f24abe7114e5dc15985, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/162fe23cc0bc4130ade01428fcdf122c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=41.6 K 2024-11-11T23:55:45,880 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d2b9ca351a454e8a7dc60b75627521, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731369342246 2024-11-11T23:55:45,880 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting abe39a39fb0b42728657ee1e7080bb96, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731369342246 2024-11-11T23:55:45,881 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c9223614c440c790e00f7a67a8dcdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731369342916 2024-11-11T23:55:45,881 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f05c46944624f24abe7114e5dc15985, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731369342916 2024-11-11T23:55:45,881 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 099bb9d6dff7499f8cddbde8be135c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1731369344152 2024-11-11T23:55:45,881 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 162fe23cc0bc4130ade01428fcdf122c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1731369344130 2024-11-11T23:55:45,890 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:45,890 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:45,891 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/ced0e6fb0e414da6ae9a29288b575d75 is 50, key is test_row_0/A:col10/1731369344154/Put/seqid=0 2024-11-11T23:55:45,891 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/9940e07b54bd4a7bb347d858da0e3c7a is 50, key is test_row_0/B:col10/1731369344154/Put/seqid=0 2024-11-11T23:55:45,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742318_1494 (size=13255) 2024-11-11T23:55:45,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742319_1495 (size=13255) 2024-11-11T23:55:45,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T23:55:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:45,986 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T23:55:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:45,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e4548d44b5c84c8189b9002cc067bcb2 is 50, key is test_row_0/A:col10/1731369345064/Put/seqid=0 2024-11-11T23:55:45,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742320_1496 (size=12301) 2024-11-11T23:55:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. as already flushing 2024-11-11T23:55:46,220 INFO [Thread-1858 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1858. 2024-11-11T23:55:46,220 DEBUG [Thread-1858 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:46,220 DEBUG [Thread-1858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:46,220 DEBUG [Thread-1858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:46,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:46,227 INFO [Thread-1856 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1856. 2024-11-11T23:55:46,227 DEBUG [Thread-1856 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:46,227 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:46,227 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:46,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T23:55:46,303 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/ced0e6fb0e414da6ae9a29288b575d75 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/ced0e6fb0e414da6ae9a29288b575d75 2024-11-11T23:55:46,304 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/9940e07b54bd4a7bb347d858da0e3c7a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9940e07b54bd4a7bb347d858da0e3c7a 2024-11-11T23:55:46,308 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/B of 6818f1e02fff6c00ff92806be77a8a11 into 9940e07b54bd4a7bb347d858da0e3c7a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:46,308 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/A of 6818f1e02fff6c00ff92806be77a8a11 into ced0e6fb0e414da6ae9a29288b575d75(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:46,308 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:46,308 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:46,309 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/B, priority=13, startTime=1731369345878; duration=0sec 2024-11-11T23:55:46,309 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/A, priority=13, startTime=1731369345878; duration=0sec 2024-11-11T23:55:46,309 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:46,309 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:46,309 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:A 2024-11-11T23:55:46,309 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:B 2024-11-11T23:55:46,309 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:55:46,309 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:55:46,310 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 6818f1e02fff6c00ff92806be77a8a11/C is initiating minor compaction (all files) 2024-11-11T23:55:46,310 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6818f1e02fff6c00ff92806be77a8a11/C in TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:46,310 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a3499ed4ad8c46008d74eb0aa94b31ef, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d52e8f2e7630448994244928fed1bd2f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/17d7c6797cdc4b949ebd7bcee6c3e6de] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp, totalSize=36.9 K 2024-11-11T23:55:46,310 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3499ed4ad8c46008d74eb0aa94b31ef, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731369342246 2024-11-11T23:55:46,310 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d52e8f2e7630448994244928fed1bd2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731369342916 2024-11-11T23:55:46,310 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17d7c6797cdc4b949ebd7bcee6c3e6de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1731369344152 2024-11-11T23:55:46,316 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6818f1e02fff6c00ff92806be77a8a11#C#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:46,316 INFO [Thread-1860 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1860. 2024-11-11T23:55:46,316 DEBUG [Thread-1860 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:46,316 DEBUG [Thread-1860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:46,316 DEBUG [Thread-1860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:46,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:46,316 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/2ec738490d2d4fa7ba9450ca113691d4 is 50, key is test_row_0/C:col10/1731369344154/Put/seqid=0 2024-11-11T23:55:46,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742321_1497 (size=13255) 2024-11-11T23:55:46,325 INFO [Thread-1854 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1854. 2024-11-11T23:55:46,325 DEBUG [Thread-1854 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:46,325 DEBUG [Thread-1854 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:46,325 DEBUG [Thread-1854 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:46,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:46,393 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e4548d44b5c84c8189b9002cc067bcb2 2024-11-11T23:55:46,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/d2c70ed6cd7c459884f6bcd7c3ac8158 is 50, key is test_row_0/B:col10/1731369345064/Put/seqid=0 2024-11-11T23:55:46,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742322_1498 (size=12301) 2024-11-11T23:55:46,724 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/2ec738490d2d4fa7ba9450ca113691d4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/2ec738490d2d4fa7ba9450ca113691d4 2024-11-11T23:55:46,727 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6818f1e02fff6c00ff92806be77a8a11/C of 6818f1e02fff6c00ff92806be77a8a11 into 2ec738490d2d4fa7ba9450ca113691d4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:46,728 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:46,728 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., storeName=6818f1e02fff6c00ff92806be77a8a11/C, priority=13, startTime=1731369345878; duration=0sec 2024-11-11T23:55:46,728 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:46,728 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6818f1e02fff6c00ff92806be77a8a11:C 2024-11-11T23:55:46,807 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/d2c70ed6cd7c459884f6bcd7c3ac8158 2024-11-11T23:55:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/12f523796ab24b9cab54a614740b5b08 is 50, key is test_row_0/C:col10/1731369345064/Put/seqid=0 2024-11-11T23:55:46,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742323_1499 (size=12301) 2024-11-11T23:55:47,225 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/12f523796ab24b9cab54a614740b5b08 2024-11-11T23:55:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e4548d44b5c84c8189b9002cc067bcb2 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e4548d44b5c84c8189b9002cc067bcb2 2024-11-11T23:55:47,241 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e4548d44b5c84c8189b9002cc067bcb2, entries=150, sequenceid=711, filesize=12.0 K 2024-11-11T23:55:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/d2c70ed6cd7c459884f6bcd7c3ac8158 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d2c70ed6cd7c459884f6bcd7c3ac8158 2024-11-11T23:55:47,245 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d2c70ed6cd7c459884f6bcd7c3ac8158, entries=150, sequenceid=711, filesize=12.0 K 2024-11-11T23:55:47,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/12f523796ab24b9cab54a614740b5b08 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/12f523796ab24b9cab54a614740b5b08 2024-11-11T23:55:47,249 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/12f523796ab24b9cab54a614740b5b08, entries=150, sequenceid=711, filesize=12.0 K 2024-11-11T23:55:47,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 6818f1e02fff6c00ff92806be77a8a11 in 1264ms, sequenceid=711, compaction requested=false 2024-11-11T23:55:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6818f1e02fff6c00ff92806be77a8a11: 2024-11-11T23:55:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T23:55:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T23:55:47,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T23:55:47,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1050 sec 2024-11-11T23:55:47,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.1090 sec 2024-11-11T23:55:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T23:55:48,325 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:48,478 INFO [Thread-1862 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1862. 2024-11-11T23:55:48,479 DEBUG [Thread-1862 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:55:48,479 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:48,479 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:48,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:48,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T23:55:48,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-11-11T23:55:48,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-11T23:55:48,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 277 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1274 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3822 rows 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1271 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3810 rows 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1263 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3786 rows 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1274 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3819 rows 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1266 2024-11-11T23:55:48,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3798 rows 2024-11-11T23:55:48,480 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:55:48,480 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:55:48,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:48,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:48,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:55:48,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T23:55:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T23:55:48,487 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369348487"}]},"ts":"1731369348487"} 2024-11-11T23:55:48,488 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T23:55:48,488 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T23:55:48,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:55:48,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, UNASSIGN}] 2024-11-11T23:55:48,491 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, UNASSIGN 2024-11-11T23:55:48,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=6818f1e02fff6c00ff92806be77a8a11, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:48,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, UNASSIGN because future has completed 2024-11-11T23:55:48,493 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:55:48,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:55:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T23:55:48,646 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:48,646 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:55:48,646 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing 6818f1e02fff6c00ff92806be77a8a11, disabling compactions & flushes 2024-11-11T23:55:48,646 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:48,646 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:48,646 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. after waiting 0 ms 2024-11-11T23:55:48,646 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:48,646 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing 6818f1e02fff6c00ff92806be77a8a11 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T23:55:48,647 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=A 2024-11-11T23:55:48,647 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:48,647 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=B 2024-11-11T23:55:48,647 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:48,647 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6818f1e02fff6c00ff92806be77a8a11, store=C 2024-11-11T23:55:48,647 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:48,652 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e14a361916624035a805eed0695b99d4 is 50, key is test_row_0/A:col10/1731369346324/Put/seqid=0 2024-11-11T23:55:48,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742324_1500 (size=12301) 2024-11-11T23:55:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T23:55:49,058 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=722 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e14a361916624035a805eed0695b99d4 2024-11-11T23:55:49,065 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/29f7ef7b61b54425a9356f5b6f61f3f2 is 50, key is test_row_0/B:col10/1731369346324/Put/seqid=0 2024-11-11T23:55:49,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742325_1501 (size=12301) 2024-11-11T23:55:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T23:55:49,470 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=722 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/29f7ef7b61b54425a9356f5b6f61f3f2 2024-11-11T23:55:49,481 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/806b39978b9a4e9eb57e5ef7891f3b20 is 50, key is test_row_0/C:col10/1731369346324/Put/seqid=0 2024-11-11T23:55:49,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742326_1502 (size=12301) 2024-11-11T23:55:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T23:55:49,731 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:55:49,887 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=722 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/806b39978b9a4e9eb57e5ef7891f3b20 2024-11-11T23:55:49,894 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/A/e14a361916624035a805eed0695b99d4 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e14a361916624035a805eed0695b99d4 2024-11-11T23:55:49,897 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e14a361916624035a805eed0695b99d4, entries=150, sequenceid=722, filesize=12.0 K 2024-11-11T23:55:49,898 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/B/29f7ef7b61b54425a9356f5b6f61f3f2 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/29f7ef7b61b54425a9356f5b6f61f3f2 2024-11-11T23:55:49,901 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/29f7ef7b61b54425a9356f5b6f61f3f2, entries=150, sequenceid=722, filesize=12.0 K 2024-11-11T23:55:49,901 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/.tmp/C/806b39978b9a4e9eb57e5ef7891f3b20 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/806b39978b9a4e9eb57e5ef7891f3b20 2024-11-11T23:55:49,904 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/806b39978b9a4e9eb57e5ef7891f3b20, entries=150, sequenceid=722, filesize=12.0 K 2024-11-11T23:55:49,905 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6818f1e02fff6c00ff92806be77a8a11 in 1259ms, sequenceid=722, compaction requested=true 2024-11-11T23:55:49,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/83a5ea4fefd841f9903cf36ebdefd486, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e185dc45db784027a260a8d0cb6e5c1c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/d7f6c7ba87e34468a81b8907fd9dbc25, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1982003f109c4138aad4ee30aa718580, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/12d578afbb1f47e18e1f77eaaf88e3f0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9504d2f09e854802879449eef06c89c4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9a9aebdd52a24c4ea71ecc734aee3096, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/39697a48142a41eaa6c2d30818601ba7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e0876e012eae451ea6397c9c90a521e1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/51136b15b66d4a9aa75180253b0df2c7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/b607d517f1284ebd9f8fa5949a8b3045, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/eb041d818b18470f9d0d0c4da235e9f9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/4f99c52d016c4cf3a58b7bb7c76e32cd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/c318d78c53a84089966d2359f6a98d19, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/55bfc38234324786a619af2218a2fa55, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e5a915dc086643fb97eb5b5367740339, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/f7df8eefcc7c4d25924a7a8c69662af5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/0871cc0e713e49b78a5e6494a3e1c07c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/71bd54fdde2b4e0091f0e992fae5e972, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/7455374b475f4b7297ddd92a5cd8aeb4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9f297c0145ef4eb29cfbe69ec90447c3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9592fc03704b457599ba76f66b3c2808, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/761cfe6aa2b8401daabea521cea0205c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/53a35d60870b47c88153d71ce92240e5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/df9dbc177ded482f8e6409c6bed33c12, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/93d2b9ca351a454e8a7dc60b75627521, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1f05c46944624f24abe7114e5dc15985, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/162fe23cc0bc4130ade01428fcdf122c] to archive 2024-11-11T23:55:49,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:55:49,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/83a5ea4fefd841f9903cf36ebdefd486 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/83a5ea4fefd841f9903cf36ebdefd486 2024-11-11T23:55:49,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e185dc45db784027a260a8d0cb6e5c1c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e185dc45db784027a260a8d0cb6e5c1c 2024-11-11T23:55:49,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/d7f6c7ba87e34468a81b8907fd9dbc25 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/d7f6c7ba87e34468a81b8907fd9dbc25 2024-11-11T23:55:49,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1982003f109c4138aad4ee30aa718580 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1982003f109c4138aad4ee30aa718580 2024-11-11T23:55:49,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/12d578afbb1f47e18e1f77eaaf88e3f0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/12d578afbb1f47e18e1f77eaaf88e3f0 2024-11-11T23:55:49,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9504d2f09e854802879449eef06c89c4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9504d2f09e854802879449eef06c89c4 2024-11-11T23:55:49,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9a9aebdd52a24c4ea71ecc734aee3096 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9a9aebdd52a24c4ea71ecc734aee3096 2024-11-11T23:55:49,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/39697a48142a41eaa6c2d30818601ba7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/39697a48142a41eaa6c2d30818601ba7 2024-11-11T23:55:49,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e0876e012eae451ea6397c9c90a521e1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e0876e012eae451ea6397c9c90a521e1 2024-11-11T23:55:49,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/51136b15b66d4a9aa75180253b0df2c7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/51136b15b66d4a9aa75180253b0df2c7 2024-11-11T23:55:49,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/b607d517f1284ebd9f8fa5949a8b3045 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/b607d517f1284ebd9f8fa5949a8b3045 2024-11-11T23:55:49,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/eb041d818b18470f9d0d0c4da235e9f9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/eb041d818b18470f9d0d0c4da235e9f9 2024-11-11T23:55:49,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/4f99c52d016c4cf3a58b7bb7c76e32cd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/4f99c52d016c4cf3a58b7bb7c76e32cd 2024-11-11T23:55:49,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/c318d78c53a84089966d2359f6a98d19 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/c318d78c53a84089966d2359f6a98d19 2024-11-11T23:55:49,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/55bfc38234324786a619af2218a2fa55 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/55bfc38234324786a619af2218a2fa55 2024-11-11T23:55:49,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e5a915dc086643fb97eb5b5367740339 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e5a915dc086643fb97eb5b5367740339 2024-11-11T23:55:49,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/f7df8eefcc7c4d25924a7a8c69662af5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/f7df8eefcc7c4d25924a7a8c69662af5 2024-11-11T23:55:49,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/0871cc0e713e49b78a5e6494a3e1c07c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/0871cc0e713e49b78a5e6494a3e1c07c 2024-11-11T23:55:49,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/71bd54fdde2b4e0091f0e992fae5e972 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/71bd54fdde2b4e0091f0e992fae5e972 2024-11-11T23:55:49,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/7455374b475f4b7297ddd92a5cd8aeb4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/7455374b475f4b7297ddd92a5cd8aeb4 2024-11-11T23:55:49,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9f297c0145ef4eb29cfbe69ec90447c3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9f297c0145ef4eb29cfbe69ec90447c3 2024-11-11T23:55:49,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9592fc03704b457599ba76f66b3c2808 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/9592fc03704b457599ba76f66b3c2808 2024-11-11T23:55:49,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/761cfe6aa2b8401daabea521cea0205c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/761cfe6aa2b8401daabea521cea0205c 2024-11-11T23:55:49,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/53a35d60870b47c88153d71ce92240e5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/53a35d60870b47c88153d71ce92240e5 2024-11-11T23:55:49,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/df9dbc177ded482f8e6409c6bed33c12 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/df9dbc177ded482f8e6409c6bed33c12 2024-11-11T23:55:49,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/93d2b9ca351a454e8a7dc60b75627521 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/93d2b9ca351a454e8a7dc60b75627521 2024-11-11T23:55:49,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1f05c46944624f24abe7114e5dc15985 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/1f05c46944624f24abe7114e5dc15985 2024-11-11T23:55:49,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/162fe23cc0bc4130ade01428fcdf122c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/162fe23cc0bc4130ade01428fcdf122c 2024-11-11T23:55:49,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f2511ccef55406c9524abf98145019d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f99c16412bac4381b342d0334dff85ac, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9bcf71eeab2742b596184f458a65fb6b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1099c313fa1c4d2b8f9ee5fed0f96cd3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/3623232015304dae91313284a6c782eb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/742bcc0fd997415a98159a3a8de470b9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/eeaced96ab184a2eafa22709b544206e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/60679a2bb5504f9796f4cefa4281bc7b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/ac664ef038dd43ce976e5f32752959ea, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a2b4b598dfcd45f7a23eabb2af6f2d89, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/c628e7ed14b54c908dec673289347a0c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4cfccf6d96e64a3991551d657d95f2b7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1bc06a60d1f04cd4b9fa29ff981c8376, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f8b10ea6ca449bf9d1fd58a52372b04, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4d6aa6e845134f819ffe597ba233e8c9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/faab52d28aff4edd9190a11b95490c7f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a4cb8a9ff41e4652a038107cb4e2306a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1f913f300cfd4f40a0bfce4ea3cce8bb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d46610944780455daff6999ffd8bcd0f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a9a2ef9dd33b4c1699a3599cc3a8ea79, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/000497c372af447298a23a7706eb7634, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/df5cf20a3e0e42f4806644024cca3f1a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d4188734fde64a92a24bcca1f1f7a8df, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f3af9b143b4e45e0812656e57d22d5e6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/abe39a39fb0b42728657ee1e7080bb96, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/311921c24d88428f99a592543357a6a5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a7c9223614c440c790e00f7a67a8dcdb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/099bb9d6dff7499f8cddbde8be135c4e] to archive 2024-11-11T23:55:49,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:55:49,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f2511ccef55406c9524abf98145019d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f2511ccef55406c9524abf98145019d 2024-11-11T23:55:49,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f99c16412bac4381b342d0334dff85ac to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f99c16412bac4381b342d0334dff85ac 2024-11-11T23:55:49,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9bcf71eeab2742b596184f458a65fb6b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9bcf71eeab2742b596184f458a65fb6b 2024-11-11T23:55:49,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1099c313fa1c4d2b8f9ee5fed0f96cd3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1099c313fa1c4d2b8f9ee5fed0f96cd3 2024-11-11T23:55:49,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/3623232015304dae91313284a6c782eb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/3623232015304dae91313284a6c782eb 2024-11-11T23:55:49,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/742bcc0fd997415a98159a3a8de470b9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/742bcc0fd997415a98159a3a8de470b9 2024-11-11T23:55:49,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/eeaced96ab184a2eafa22709b544206e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/eeaced96ab184a2eafa22709b544206e 2024-11-11T23:55:49,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/60679a2bb5504f9796f4cefa4281bc7b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/60679a2bb5504f9796f4cefa4281bc7b 2024-11-11T23:55:49,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/ac664ef038dd43ce976e5f32752959ea to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/ac664ef038dd43ce976e5f32752959ea 2024-11-11T23:55:49,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a2b4b598dfcd45f7a23eabb2af6f2d89 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a2b4b598dfcd45f7a23eabb2af6f2d89 2024-11-11T23:55:49,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/c628e7ed14b54c908dec673289347a0c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/c628e7ed14b54c908dec673289347a0c 2024-11-11T23:55:49,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4cfccf6d96e64a3991551d657d95f2b7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4cfccf6d96e64a3991551d657d95f2b7 2024-11-11T23:55:49,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1bc06a60d1f04cd4b9fa29ff981c8376 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1bc06a60d1f04cd4b9fa29ff981c8376 2024-11-11T23:55:49,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f8b10ea6ca449bf9d1fd58a52372b04 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4f8b10ea6ca449bf9d1fd58a52372b04 2024-11-11T23:55:49,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4d6aa6e845134f819ffe597ba233e8c9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/4d6aa6e845134f819ffe597ba233e8c9 2024-11-11T23:55:49,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/faab52d28aff4edd9190a11b95490c7f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/faab52d28aff4edd9190a11b95490c7f 2024-11-11T23:55:49,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a4cb8a9ff41e4652a038107cb4e2306a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a4cb8a9ff41e4652a038107cb4e2306a 2024-11-11T23:55:49,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1f913f300cfd4f40a0bfce4ea3cce8bb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/1f913f300cfd4f40a0bfce4ea3cce8bb 2024-11-11T23:55:49,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d46610944780455daff6999ffd8bcd0f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d46610944780455daff6999ffd8bcd0f 2024-11-11T23:55:49,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a9a2ef9dd33b4c1699a3599cc3a8ea79 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a9a2ef9dd33b4c1699a3599cc3a8ea79 2024-11-11T23:55:49,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/000497c372af447298a23a7706eb7634 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/000497c372af447298a23a7706eb7634 2024-11-11T23:55:49,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/df5cf20a3e0e42f4806644024cca3f1a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/df5cf20a3e0e42f4806644024cca3f1a 2024-11-11T23:55:49,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d4188734fde64a92a24bcca1f1f7a8df to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d4188734fde64a92a24bcca1f1f7a8df 2024-11-11T23:55:49,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f3af9b143b4e45e0812656e57d22d5e6 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/f3af9b143b4e45e0812656e57d22d5e6 2024-11-11T23:55:49,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/abe39a39fb0b42728657ee1e7080bb96 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/abe39a39fb0b42728657ee1e7080bb96 2024-11-11T23:55:49,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/311921c24d88428f99a592543357a6a5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/311921c24d88428f99a592543357a6a5 2024-11-11T23:55:49,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a7c9223614c440c790e00f7a67a8dcdb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/a7c9223614c440c790e00f7a67a8dcdb 2024-11-11T23:55:49,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/099bb9d6dff7499f8cddbde8be135c4e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/099bb9d6dff7499f8cddbde8be135c4e 2024-11-11T23:55:49,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a797629be6624eb497ba45b23a2ee4d5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/59c7ac7567e34fca991bc3001e030a81, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/bf567b4e6420493b84568da50ecabf70, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4a791bee11de4885a30cfdf258109d40, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9900e905326a4307b842647bd16ba364, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9e94a36114e641b29e1393cea9c4badb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/aae8264be3c24955bb680c66ef655a33, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/405af0c5a66b486d83cd933fc53635da, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/93537cd0991f47a599d82f0272651fe6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/2e28ea27b7af4a4da9f4b83096d4e177, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/048339648a6747c0b01a2b9f0078a66e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/3a9087fb5b43406eb4d61a1e575abd09, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4586b410c74e4b6ebef801e9f2214d6e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/06120162ca9a44c3970fcaa189cae7e4, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89c427b05cfc4484a47688a4bbafddcb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b5e6da1c754c42cf92b6b4381c0aff8d, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/11c235923435469d82629d82da5b51f3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/10598733fa41430b87123d0666e59904, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/e5489b813fca4f7bb2652d49fbecac30, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d0142cb92aa74143a15ed391dcd137a7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b827db4fb0c54859a430f38c886a3a56, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/25ee6d63bec54ef780fd255c020cb4e5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89519ce01ff7407496d235b0b501ca32, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4ae6a8a8e4f04f1480e455d19e186c4e, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a3499ed4ad8c46008d74eb0aa94b31ef, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/634e3b4194344988bd25524669db2738, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d52e8f2e7630448994244928fed1bd2f, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/17d7c6797cdc4b949ebd7bcee6c3e6de] to archive 2024-11-11T23:55:49,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:55:49,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a797629be6624eb497ba45b23a2ee4d5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a797629be6624eb497ba45b23a2ee4d5 2024-11-11T23:55:49,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/59c7ac7567e34fca991bc3001e030a81 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/59c7ac7567e34fca991bc3001e030a81 2024-11-11T23:55:49,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/bf567b4e6420493b84568da50ecabf70 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/bf567b4e6420493b84568da50ecabf70 2024-11-11T23:55:49,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4a791bee11de4885a30cfdf258109d40 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4a791bee11de4885a30cfdf258109d40 2024-11-11T23:55:49,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9900e905326a4307b842647bd16ba364 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9900e905326a4307b842647bd16ba364 2024-11-11T23:55:49,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9e94a36114e641b29e1393cea9c4badb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/9e94a36114e641b29e1393cea9c4badb 2024-11-11T23:55:49,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/aae8264be3c24955bb680c66ef655a33 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/aae8264be3c24955bb680c66ef655a33 2024-11-11T23:55:49,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/405af0c5a66b486d83cd933fc53635da to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/405af0c5a66b486d83cd933fc53635da 2024-11-11T23:55:49,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/93537cd0991f47a599d82f0272651fe6 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/93537cd0991f47a599d82f0272651fe6 2024-11-11T23:55:49,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/2e28ea27b7af4a4da9f4b83096d4e177 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/2e28ea27b7af4a4da9f4b83096d4e177 2024-11-11T23:55:49,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/048339648a6747c0b01a2b9f0078a66e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/048339648a6747c0b01a2b9f0078a66e 2024-11-11T23:55:49,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/3a9087fb5b43406eb4d61a1e575abd09 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/3a9087fb5b43406eb4d61a1e575abd09 2024-11-11T23:55:49,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4586b410c74e4b6ebef801e9f2214d6e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4586b410c74e4b6ebef801e9f2214d6e 2024-11-11T23:55:49,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/06120162ca9a44c3970fcaa189cae7e4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/06120162ca9a44c3970fcaa189cae7e4 2024-11-11T23:55:49,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89c427b05cfc4484a47688a4bbafddcb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89c427b05cfc4484a47688a4bbafddcb 2024-11-11T23:55:49,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b5e6da1c754c42cf92b6b4381c0aff8d to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b5e6da1c754c42cf92b6b4381c0aff8d 2024-11-11T23:55:49,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/11c235923435469d82629d82da5b51f3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/11c235923435469d82629d82da5b51f3 2024-11-11T23:55:49,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/10598733fa41430b87123d0666e59904 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/10598733fa41430b87123d0666e59904 2024-11-11T23:55:49,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/e5489b813fca4f7bb2652d49fbecac30 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/e5489b813fca4f7bb2652d49fbecac30 2024-11-11T23:55:49,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d0142cb92aa74143a15ed391dcd137a7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d0142cb92aa74143a15ed391dcd137a7 2024-11-11T23:55:49,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b827db4fb0c54859a430f38c886a3a56 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/b827db4fb0c54859a430f38c886a3a56 2024-11-11T23:55:49,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/25ee6d63bec54ef780fd255c020cb4e5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/25ee6d63bec54ef780fd255c020cb4e5 2024-11-11T23:55:49,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89519ce01ff7407496d235b0b501ca32 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/89519ce01ff7407496d235b0b501ca32 2024-11-11T23:55:49,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4ae6a8a8e4f04f1480e455d19e186c4e to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/4ae6a8a8e4f04f1480e455d19e186c4e 2024-11-11T23:55:50,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a3499ed4ad8c46008d74eb0aa94b31ef to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/a3499ed4ad8c46008d74eb0aa94b31ef 2024-11-11T23:55:50,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/634e3b4194344988bd25524669db2738 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/634e3b4194344988bd25524669db2738 2024-11-11T23:55:50,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d52e8f2e7630448994244928fed1bd2f to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/d52e8f2e7630448994244928fed1bd2f 2024-11-11T23:55:50,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/17d7c6797cdc4b949ebd7bcee6c3e6de to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/17d7c6797cdc4b949ebd7bcee6c3e6de 2024-11-11T23:55:50,007 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/recovered.edits/725.seqid, newMaxSeqId=725, maxSeqId=1 2024-11-11T23:55:50,007 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. 2024-11-11T23:55:50,007 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for 6818f1e02fff6c00ff92806be77a8a11: Waiting for close lock at 1731369348646Running coprocessor pre-close hooks at 1731369348646Disabling compacts and flushes for region at 1731369348646Disabling writes for close at 1731369348646Obtaining lock to block concurrent updates at 1731369348646Preparing flush snapshotting stores in 6818f1e02fff6c00ff92806be77a8a11 at 1731369348646Finished memstore snapshotting TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731369348647 (+1 ms)Flushing stores of TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11. at 1731369348648 (+1 ms)Flushing 6818f1e02fff6c00ff92806be77a8a11/A: creating writer at 1731369348648Flushing 6818f1e02fff6c00ff92806be77a8a11/A: appending metadata at 1731369348652 (+4 ms)Flushing 6818f1e02fff6c00ff92806be77a8a11/A: closing flushed file at 1731369348652Flushing 6818f1e02fff6c00ff92806be77a8a11/B: creating writer at 1731369349063 (+411 ms)Flushing 6818f1e02fff6c00ff92806be77a8a11/B: appending metadata at 1731369349065 (+2 ms)Flushing 6818f1e02fff6c00ff92806be77a8a11/B: closing flushed file at 1731369349065Flushing 6818f1e02fff6c00ff92806be77a8a11/C: creating writer at 1731369349478 (+413 ms)Flushing 6818f1e02fff6c00ff92806be77a8a11/C: appending metadata at 1731369349480 (+2 ms)Flushing 6818f1e02fff6c00ff92806be77a8a11/C: closing flushed file at 1731369349481 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50993a74: reopening flushed file at 1731369349894 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ef718c7: reopening flushed file at 1731369349897 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aa19958: reopening flushed file at 1731369349901 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6818f1e02fff6c00ff92806be77a8a11 in 1259ms, sequenceid=722, compaction requested=true at 1731369349905 (+4 ms)Writing region close event to WAL at 1731369350004 (+99 ms)Running coprocessor post-close hooks at 1731369350007 (+3 ms)Closed at 1731369350007 2024-11-11T23:55:50,009 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed 6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:50,010 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=6818f1e02fff6c00ff92806be77a8a11, regionState=CLOSED 2024-11-11T23:55:50,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:55:50,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T23:55:50,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure 6818f1e02fff6c00ff92806be77a8a11, server=b70d29849faf,42393,1731369201430 in 1.5180 sec 2024-11-11T23:55:50,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-11T23:55:50,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6818f1e02fff6c00ff92806be77a8a11, UNASSIGN in 1.5230 sec 2024-11-11T23:55:50,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T23:55:50,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5250 sec 2024-11-11T23:55:50,017 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369350016"}]},"ts":"1731369350016"} 2024-11-11T23:55:50,018 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T23:55:50,018 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T23:55:50,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5360 sec 2024-11-11T23:55:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T23:55:50,637 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:50,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T23:55:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:50,639 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T23:55:50,640 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:50,641 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:50,643 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/recovered.edits] 2024-11-11T23:55:50,646 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/ced0e6fb0e414da6ae9a29288b575d75 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/ced0e6fb0e414da6ae9a29288b575d75 2024-11-11T23:55:50,647 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e14a361916624035a805eed0695b99d4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e14a361916624035a805eed0695b99d4 2024-11-11T23:55:50,648 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e4548d44b5c84c8189b9002cc067bcb2 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/A/e4548d44b5c84c8189b9002cc067bcb2 2024-11-11T23:55:50,649 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/29f7ef7b61b54425a9356f5b6f61f3f2 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/29f7ef7b61b54425a9356f5b6f61f3f2 2024-11-11T23:55:50,650 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9940e07b54bd4a7bb347d858da0e3c7a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/9940e07b54bd4a7bb347d858da0e3c7a 2024-11-11T23:55:50,651 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d2c70ed6cd7c459884f6bcd7c3ac8158 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/B/d2c70ed6cd7c459884f6bcd7c3ac8158 2024-11-11T23:55:50,653 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/12f523796ab24b9cab54a614740b5b08 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/12f523796ab24b9cab54a614740b5b08 2024-11-11T23:55:50,654 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/2ec738490d2d4fa7ba9450ca113691d4 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/2ec738490d2d4fa7ba9450ca113691d4 2024-11-11T23:55:50,654 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/806b39978b9a4e9eb57e5ef7891f3b20 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/C/806b39978b9a4e9eb57e5ef7891f3b20 2024-11-11T23:55:50,656 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/recovered.edits/725.seqid to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11/recovered.edits/725.seqid 2024-11-11T23:55:50,656 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/6818f1e02fff6c00ff92806be77a8a11 2024-11-11T23:55:50,657 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T23:55:50,658 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:50,660 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T23:55:50,661 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T23:55:50,662 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:50,662 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T23:55:50,662 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731369350662"}]},"ts":"9223372036854775807"} 2024-11-11T23:55:50,664 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T23:55:50,664 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6818f1e02fff6c00ff92806be77a8a11, NAME => 'TestAcidGuarantees,,1731369323248.6818f1e02fff6c00ff92806be77a8a11.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T23:55:50,664 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T23:55:50,664 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731369350664"}]},"ts":"9223372036854775807"} 2024-11-11T23:55:50,665 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T23:55:50,666 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:50,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-11T23:55:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T23:55:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T23:55:50,746 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:50,752 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 251) - Thread LEAK? -, OpenFileDescriptor=457 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=364 (was 385), ProcessCount=11 (was 11), AvailableMemoryMB=4447 (was 4586) 2024-11-11T23:55:50,759 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=364, ProcessCount=11, AvailableMemoryMB=4447 2024-11-11T23:55:50,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:55:50,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:50,762 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:50,763 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:50,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-11-11T23:55:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T23:55:50,763 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:50,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742327_1503 (size=1061) 2024-11-11T23:55:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T23:55:51,037 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T23:55:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T23:55:51,171 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075 2024-11-11T23:55:51,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742328_1504 (size=53) 2024-11-11T23:55:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T23:55:51,579 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:51,580 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 25afeec37b6a72df3a36de94b90eba91, disabling compactions & flushes 2024-11-11T23:55:51,580 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:51,580 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:51,580 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. after waiting 0 ms 2024-11-11T23:55:51,580 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:51,580 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:51,580 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 25afeec37b6a72df3a36de94b90eba91: Waiting for close lock at 1731369351579Disabling compacts and flushes for region at 1731369351579Disabling writes for close at 1731369351580 (+1 ms)Writing region close event to WAL at 1731369351580Closed at 1731369351580 2024-11-11T23:55:51,582 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:51,583 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731369351582"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369351582"}]},"ts":"1731369351582"} 2024-11-11T23:55:51,585 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:51,587 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:51,587 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369351587"}]},"ts":"1731369351587"} 2024-11-11T23:55:51,589 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T23:55:51,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, ASSIGN}] 2024-11-11T23:55:51,592 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, ASSIGN 2024-11-11T23:55:51,592 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, ASSIGN; state=OFFLINE, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=false 2024-11-11T23:55:51,743 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=25afeec37b6a72df3a36de94b90eba91, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:51,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, ASSIGN because future has completed 2024-11-11T23:55:51,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:55:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T23:55:51,908 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:51,908 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:51,908 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,909 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:51,909 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,909 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,912 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,914 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:51,914 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25afeec37b6a72df3a36de94b90eba91 columnFamilyName A 2024-11-11T23:55:51,914 DEBUG [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:51,915 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(327): Store=25afeec37b6a72df3a36de94b90eba91/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:51,915 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,917 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:51,917 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25afeec37b6a72df3a36de94b90eba91 columnFamilyName B 2024-11-11T23:55:51,917 DEBUG [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:51,917 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(327): Store=25afeec37b6a72df3a36de94b90eba91/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:51,917 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,918 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:51,918 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25afeec37b6a72df3a36de94b90eba91 columnFamilyName C 2024-11-11T23:55:51,918 DEBUG [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:51,919 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(327): Store=25afeec37b6a72df3a36de94b90eba91/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:51,919 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:51,919 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,919 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,920 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,920 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,920 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,921 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:55:51,922 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,923 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:51,923 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened 25afeec37b6a72df3a36de94b90eba91; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67912693, jitterRate=0.011977985501289368}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:55:51,923 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:51,924 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for 25afeec37b6a72df3a36de94b90eba91: Running coprocessor pre-open hook at 1731369351909Writing region info on filesystem at 1731369351909Initializing all the Stores at 1731369351911 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369351911Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369351911Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369351911Cleaning up temporary data from old regions at 1731369351920 (+9 ms)Running coprocessor post-open hooks at 1731369351923 (+3 ms)Region opened successfully at 1731369351924 (+1 ms) 2024-11-11T23:55:51,925 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., pid=136, masterSystemTime=1731369351901 2024-11-11T23:55:51,926 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:51,926 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:51,926 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=25afeec37b6a72df3a36de94b90eba91, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:51,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:55:51,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T23:55:51,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 in 181 msec 2024-11-11T23:55:51,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-11T23:55:51,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, ASSIGN in 340 msec 2024-11-11T23:55:51,932 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:51,932 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369351932"}]},"ts":"1731369351932"} 2024-11-11T23:55:51,933 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T23:55:51,934 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:51,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-11-11T23:55:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T23:55:52,915 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:52,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:52,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:52,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:52,918 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@968331d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:52,921 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b08318c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:52,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:52,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:52,926 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T23:55:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b70d29849faf,37327,1731369200577 2024-11-11T23:55:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5dec2327 2024-11-11T23:55:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T23:55:52,929 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T23:55:52,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T23:55:52,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T23:55:52,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742329_1505 (size=1097) 2024-11-11T23:55:53,349 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T23:55:53,349 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T23:55:53,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:55:53,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, REOPEN/MOVE}] 2024-11-11T23:55:53,358 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, REOPEN/MOVE 2024-11-11T23:55:53,359 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=25afeec37b6a72df3a36de94b90eba91, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:53,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, REOPEN/MOVE because future has completed 2024-11-11T23:55:53,361 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:55:53,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:55:53,514 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,514 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:55:53,515 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 25afeec37b6a72df3a36de94b90eba91, disabling compactions & flushes 2024-11-11T23:55:53,515 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:53,515 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:53,515 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. after waiting 0 ms 2024-11-11T23:55:53,515 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:53,524 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T23:55:53,525 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:53,525 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 25afeec37b6a72df3a36de94b90eba91: Waiting for close lock at 1731369353515Running coprocessor pre-close hooks at 1731369353515Disabling compacts and flushes for region at 1731369353515Disabling writes for close at 1731369353515Writing region close event to WAL at 1731369353516 (+1 ms)Running coprocessor post-close hooks at 1731369353525 (+9 ms)Closed at 1731369353525 2024-11-11T23:55:53,525 WARN [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: 25afeec37b6a72df3a36de94b90eba91 to self. 2024-11-11T23:55:53,527 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,528 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=25afeec37b6a72df3a36de94b90eba91, regionState=CLOSED 2024-11-11T23:55:53,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:55:53,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T23:55:53,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 in 169 msec 2024-11-11T23:55:53,532 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, REOPEN/MOVE; state=CLOSED, location=b70d29849faf,42393,1731369201430; forceNewPlan=false, retain=true 2024-11-11T23:55:53,683 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=25afeec37b6a72df3a36de94b90eba91, regionState=OPENING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:53,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, REOPEN/MOVE because future has completed 2024-11-11T23:55:53,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:55:53,848 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:53,848 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:53,849 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,849 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:53,849 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,849 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,850 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,850 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:53,851 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25afeec37b6a72df3a36de94b90eba91 columnFamilyName A 2024-11-11T23:55:53,852 DEBUG [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:53,852 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(327): Store=25afeec37b6a72df3a36de94b90eba91/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:53,852 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,853 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:53,853 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25afeec37b6a72df3a36de94b90eba91 columnFamilyName B 2024-11-11T23:55:53,853 DEBUG [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:53,853 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(327): Store=25afeec37b6a72df3a36de94b90eba91/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:53,853 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,854 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T23:55:53,854 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25afeec37b6a72df3a36de94b90eba91 columnFamilyName C 2024-11-11T23:55:53,854 DEBUG [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:53,854 INFO [StoreOpener-25afeec37b6a72df3a36de94b90eba91-1 {}] regionserver.HStore(327): Store=25afeec37b6a72df3a36de94b90eba91/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:53,854 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:53,854 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,855 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,856 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,856 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,856 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,857 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T23:55:53,858 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,858 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened 25afeec37b6a72df3a36de94b90eba91; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67194656, jitterRate=0.0012784004211425781}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T23:55:53,859 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,859 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for 25afeec37b6a72df3a36de94b90eba91: Running coprocessor pre-open hook at 1731369353849Writing region info on filesystem at 1731369353849Initializing all the Stores at 1731369353850 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369353850Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369353850Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369353850Cleaning up temporary data from old regions at 1731369353856 (+6 ms)Running coprocessor post-open hooks at 1731369353859 (+3 ms)Region opened successfully at 1731369353859 2024-11-11T23:55:53,860 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., pid=141, masterSystemTime=1731369353841 2024-11-11T23:55:53,862 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:53,862 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:53,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=25afeec37b6a72df3a36de94b90eba91, regionState=OPEN, openSeqNum=5, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:55:53,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:55:53,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-11-11T23:55:53,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 in 177 msec 2024-11-11T23:55:53,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-11T23:55:53,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, REOPEN/MOVE in 509 msec 2024-11-11T23:55:53,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T23:55:53,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-11-11T23:55:53,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-11-11T23:55:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T23:55:53,872 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:53,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,874 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab55dc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d15d204, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,877 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511f8fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c1e5c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,879 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45301490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7a5db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,882 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6453f051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5401f36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309489e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1518a95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d1a5e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ce8ea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c526880, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,892 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d705bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,894 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30babd2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,895 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441307e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4738b636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f3eb234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,37327,-1 for getting cluster id 2024-11-11T23:55:53,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:53,900 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8859011-a7e7-41c7-b37b-4a7d6cebf423' 2024-11-11T23:55:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8859011-a7e7-41c7-b37b-4a7d6cebf423" 2024-11-11T23:55:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7c581c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,37327,-1] 2024-11-11T23:55:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:53,901 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5573d994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:53,902 DEBUG [Thread-2276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,902 DEBUG [Thread-2282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,902 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,902 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,903 DEBUG [Thread-2265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,903 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-11T23:55:53,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,906 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T23:55:53,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,907 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:53,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,908 DEBUG [Thread-2284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,911 DEBUG [Thread-2271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T23:55:53,917 DEBUG [Thread-2269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,918 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,42393,1731369201430, seqNum=-1] 2024-11-11T23:55:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:53,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:55:53,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:53,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:55:53,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:53,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:55:53,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:53,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5] 2024-11-11T23:55:53,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e449d1cbf3354adfa65c176f6b0ddeed_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369353917/Put/seqid=0 2024-11-11T23:55:53,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369413942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:53,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369353940, max timestamp=1731369353947Num uniques 859; 2024-11-11T23:55:53,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369353940, max timestamp=1731369353947Num uniques 859; 2024-11-11T23:55:53,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:53,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:53,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742330_1506 (size=9714) 2024-11-11T23:55:53,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731369353940, max timestamp=1731369353947Num uniques 859; 2024-11-11T23:55:53,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T23:55:53,950 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:53,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e449d1cbf3354adfa65c176f6b0ddeed_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e449d1cbf3354adfa65c176f6b0ddeed_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:53,954 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/2beaf225d8244372a53caf2e7b45163b, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:53,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/2beaf225d8244372a53caf2e7b45163b is 175, key is test_row_0/A:col10/1731369353917/Put/seqid=0 2024-11-11T23:55:53,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742331_1507 (size=22361) 2024-11-11T23:55:53,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/2beaf225d8244372a53caf2e7b45163b 2024-11-11T23:55:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369413973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:53,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369413977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:53,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369413977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:53,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369413978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:53,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/f3220f7c58b243efa22ec2d82e3b4c25 is 50, key is test_row_0/B:col10/1731369353917/Put/seqid=0 2024-11-11T23:55:53,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742332_1508 (size=9657) 2024-11-11T23:55:53,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/f3220f7c58b243efa22ec2d82e3b4c25 2024-11-11T23:55:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T23:55:54,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/2b87b943658446e2974dddf3a1d536ef is 50, key is test_row_0/C:col10/1731369353917/Put/seqid=0 2024-11-11T23:55:54,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742333_1509 (size=9657) 2024-11-11T23:55:54,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/2b87b943658446e2974dddf3a1d536ef 2024-11-11T23:55:54,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/2beaf225d8244372a53caf2e7b45163b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/2beaf225d8244372a53caf2e7b45163b 2024-11-11T23:55:54,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/2beaf225d8244372a53caf2e7b45163b, entries=100, sequenceid=15, filesize=21.8 K 2024-11-11T23:55:54,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/f3220f7c58b243efa22ec2d82e3b4c25 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f3220f7c58b243efa22ec2d82e3b4c25 2024-11-11T23:55:54,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f3220f7c58b243efa22ec2d82e3b4c25, entries=100, sequenceid=15, filesize=9.4 K 2024-11-11T23:55:54,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/2b87b943658446e2974dddf3a1d536ef as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2b87b943658446e2974dddf3a1d536ef 2024-11-11T23:55:54,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2b87b943658446e2974dddf3a1d536ef, entries=100, sequenceid=15, filesize=9.4 K 2024-11-11T23:55:54,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for 25afeec37b6a72df3a36de94b90eba91 in 129ms, sequenceid=15, compaction requested=false 2024-11-11T23:55:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:55:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:54,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=143.13 KB heapSize=410.69 KB 2024-11-11T23:55:54,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:55:54,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:54,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:55:54,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:54,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:55:54,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T23:55:54,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T23:55:54,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:54,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111144b878e8358a422982c448d1b3364210_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369353976/Put/seqid=0 2024-11-11T23:55:54,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742334_1510 (size=14594) 2024-11-11T23:55:54,074 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:54,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111144b878e8358a422982c448d1b3364210_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111144b878e8358a422982c448d1b3364210_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:54,079 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/d0a971a32aec4a7ebeae22050ad0199c, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:54,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/d0a971a32aec4a7ebeae22050ad0199c is 175, key is test_row_0/A:col10/1731369353976/Put/seqid=0 2024-11-11T23:55:54,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369414085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369414085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742335_1511 (size=39549) 2024-11-11T23:55:54,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/d0a971a32aec4a7ebeae22050ad0199c 2024-11-11T23:55:54,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369414088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369414094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369414094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/d87bc71472e241159709a572e49d339a is 50, key is test_row_0/B:col10/1731369353976/Put/seqid=0 2024-11-11T23:55:54,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742336_1512 (size=12001) 2024-11-11T23:55:54,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369414195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T23:55:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T23:55:54,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369414295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369414295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369414304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369414304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T23:55:54,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:54,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369414404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T23:55:54,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:54,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/d87bc71472e241159709a572e49d339a 2024-11-11T23:55:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T23:55:54,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/da3bb0a943804afbaf534cfc3f123099 is 50, key is test_row_0/C:col10/1731369353976/Put/seqid=0 2024-11-11T23:55:54,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742337_1513 (size=12001) 2024-11-11T23:55:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369414605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369414605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369414614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369414614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T23:55:54,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:54,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369414715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:54,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T23:55:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:54,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/da3bb0a943804afbaf534cfc3f123099 2024-11-11T23:55:54,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/d0a971a32aec4a7ebeae22050ad0199c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d0a971a32aec4a7ebeae22050ad0199c 2024-11-11T23:55:54,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d0a971a32aec4a7ebeae22050ad0199c, entries=200, sequenceid=53, filesize=38.6 K 2024-11-11T23:55:54,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/d87bc71472e241159709a572e49d339a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/d87bc71472e241159709a572e49d339a 2024-11-11T23:55:54,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/d87bc71472e241159709a572e49d339a, entries=150, sequenceid=53, filesize=11.7 K 2024-11-11T23:55:54,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/da3bb0a943804afbaf534cfc3f123099 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/da3bb0a943804afbaf534cfc3f123099 2024-11-11T23:55:54,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/da3bb0a943804afbaf534cfc3f123099, entries=150, sequenceid=53, filesize=11.7 K 2024-11-11T23:55:54,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~410.64 KB/420496, currentSize=40.25 KB/41220 for 25afeec37b6a72df3a36de94b90eba91 in 900ms, sequenceid=53, compaction requested=false 2024-11-11T23:55:54,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:55:54,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T23:55:54,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:54,975 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T23:55:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:55:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:55:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:55:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:54,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113714a482f8d24620a4f897dac71475c1_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369354087/Put/seqid=0 2024-11-11T23:55:54,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742338_1514 (size=12154) 2024-11-11T23:55:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T23:55:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:55,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:55,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369355132, max timestamp=1731369355139Num uniques 859; 2024-11-11T23:55:55,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:55:55,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369355132, max timestamp=1731369355139Num uniques 859; 2024-11-11T23:55:55,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:55:55,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731369355132, max timestamp=1731369355139Num uniques 859; 2024-11-11T23:55:55,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T23:55:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369415159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369415161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369415162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369415163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369415224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369415274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369415275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369415275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369415275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:55,387 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113714a482f8d24620a4f897dac71475c1_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113714a482f8d24620a4f897dac71475c1_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:55,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/54fef06bb67545319f0c3485bec616e6, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/54fef06bb67545319f0c3485bec616e6 is 175, key is test_row_0/A:col10/1731369354087/Put/seqid=0 2024-11-11T23:55:55,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742339_1515 (size=30955) 2024-11-11T23:55:55,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369415484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369415485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369415485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369415485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,531 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:55:55,791 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/54fef06bb67545319f0c3485bec616e6 2024-11-11T23:55:55,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/eee6912e7c244fe9af80e325e9881bd8 is 50, key is test_row_0/B:col10/1731369354087/Put/seqid=0 2024-11-11T23:55:55,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369415794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369415794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369415794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369415795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:55,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742340_1516 (size=12001) 2024-11-11T23:55:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T23:55:56,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/eee6912e7c244fe9af80e325e9881bd8 2024-11-11T23:55:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/81f6c9400d39421f84ac1d15df118b87 is 50, key is test_row_0/C:col10/1731369354087/Put/seqid=0 2024-11-11T23:55:56,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742341_1517 (size=12001) 2024-11-11T23:55:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369416244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:56,248 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369416304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369416304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369416304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369416304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:56,609 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/81f6c9400d39421f84ac1d15df118b87 2024-11-11T23:55:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/54fef06bb67545319f0c3485bec616e6 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/54fef06bb67545319f0c3485bec616e6 2024-11-11T23:55:56,615 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/54fef06bb67545319f0c3485bec616e6, entries=150, sequenceid=62, filesize=30.2 K 2024-11-11T23:55:56,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/eee6912e7c244fe9af80e325e9881bd8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/eee6912e7c244fe9af80e325e9881bd8 2024-11-11T23:55:56,619 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/eee6912e7c244fe9af80e325e9881bd8, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T23:55:56,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/81f6c9400d39421f84ac1d15df118b87 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/81f6c9400d39421f84ac1d15df118b87 2024-11-11T23:55:56,622 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/81f6c9400d39421f84ac1d15df118b87, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T23:55:56,623 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 25afeec37b6a72df3a36de94b90eba91 in 1648ms, sequenceid=62, compaction requested=true 2024-11-11T23:55:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:55:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-11T23:55:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-11T23:55:56,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-11T23:55:56,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7170 sec 2024-11-11T23:55:56,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.7240 sec 2024-11-11T23:55:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:57,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-11T23:55:57,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369355154, max timestamp=1731369357317Num uniques 1118; 2024-11-11T23:55:57,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-11T23:55:57,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369355154, max timestamp=1731369357317Num uniques 1118; 2024-11-11T23:55:57,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-11T23:55:57,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:55:57,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:57,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731369355154, max timestamp=1731369357317Num uniques 1118; 2024-11-11T23:55:57,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:55:57,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:57,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-11T23:55:57,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:55:57,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:57,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d77da99b20c6448bb60bc98447227f1b_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369357317/Put/seqid=0 2024-11-11T23:55:57,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742342_1518 (size=14594) 2024-11-11T23:55:57,337 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:57,340 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d77da99b20c6448bb60bc98447227f1b_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d77da99b20c6448bb60bc98447227f1b_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:57,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/d4a8bb1c50e44329b67c49d1bcb7394a, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:57,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/d4a8bb1c50e44329b67c49d1bcb7394a is 175, key is test_row_0/A:col10/1731369357317/Put/seqid=0 2024-11-11T23:55:57,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369357338, max timestamp=1731369357350Num uniques 859; 2024-11-11T23:55:57,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:57,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369357338, max timestamp=1731369357350Num uniques 859; 2024-11-11T23:55:57,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731369357338, max timestamp=1731369357350Num uniques 859; 2024-11-11T23:55:57,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:57,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T23:55:57,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742343_1519 (size=39549) 2024-11-11T23:55:57,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369417375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369357366, max timestamp=1731369357377Num uniques 1068; 2024-11-11T23:55:57,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369357366, max timestamp=1731369357377Num uniques 1068; 2024-11-11T23:55:57,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T23:55:57,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T23:55:57,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731369357366, max timestamp=1731369357377Num uniques 1068; 2024-11-11T23:55:57,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T23:55:57,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369357402, max timestamp=1731369357414Num uniques 1027; 2024-11-11T23:55:57,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369357402, max timestamp=1731369357414Num uniques 1027; 2024-11-11T23:55:57,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:57,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:57,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731369357402, max timestamp=1731369357414Num uniques 1027; 2024-11-11T23:55:57,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T23:55:57,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731369357440, max timestamp=1731369357454Num uniques 1036; 2024-11-11T23:55:57,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-11T23:55:57,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731369357440, max timestamp=1731369357454Num uniques 1036; 2024-11-11T23:55:57,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-11T23:55:57,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731369357440, max timestamp=1731369357454Num uniques 1036; 2024-11-11T23:55:57,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-11T23:55:57,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731369357479, max timestamp=1731369357491Num uniques 1095; 2024-11-11T23:55:57,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-11T23:55:57,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731369357479, max timestamp=1731369357491Num uniques 1095; 2024-11-11T23:55:57,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-11T23:55:57,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731369357479, max timestamp=1731369357491Num uniques 1095; 2024-11-11T23:55:57,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-11T23:55:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731369357515, max timestamp=1731369357526Num uniques 1104; 2024-11-11T23:55:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731369357515, max timestamp=1731369357526Num uniques 1104; 2024-11-11T23:55:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-11T23:55:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-11T23:55:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731369357515, max timestamp=1731369357526Num uniques 1104; 2024-11-11T23:55:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-11T23:55:57,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731369357544, max timestamp=1731369357559Num uniques 1063; 2024-11-11T23:55:57,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-11T23:55:57,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731369357544, max timestamp=1731369357559Num uniques 1063; 2024-11-11T23:55:57,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-11T23:55:57,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731369357544, max timestamp=1731369357559Num uniques 1063; 2024-11-11T23:55:57,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-11T23:55:57,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369417588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731369357576, max timestamp=1731369357591Num uniques 1072; 2024-11-11T23:55:57,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T23:55:57,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731369357576, max timestamp=1731369357591Num uniques 1072; 2024-11-11T23:55:57,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T23:55:57,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731369357576, max timestamp=1731369357591Num uniques 1072; 2024-11-11T23:55:57,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T23:55:57,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731369357599, max timestamp=1731369357624Num uniques 1081; 2024-11-11T23:55:57,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-11T23:55:57,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731369357599, max timestamp=1731369357624Num uniques 1081; 2024-11-11T23:55:57,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-11T23:55:57,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731369357599, max timestamp=1731369357624Num uniques 1081; 2024-11-11T23:55:57,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-11T23:55:57,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731369357646, max timestamp=1731369357658Num uniques 1040; 2024-11-11T23:55:57,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=13280, suffix heap size=136128,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=290 2024-11-11T23:55:57,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731369357646, max timestamp=1731369357658Num uniques 1090; 2024-11-11T23:55:57,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-11T23:55:57,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731369357646, max timestamp=1731369357658Num uniques 1090; 2024-11-11T23:55:57,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-11T23:55:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369417688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369417689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731369357680, max timestamp=1731369357691Num uniques 1149; 2024-11-11T23:55:57,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=9114, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-11T23:55:57,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731369357680, max timestamp=1731369357691Num uniques 1149; 2024-11-11T23:55:57,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=9114, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-11T23:55:57,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731369357680, max timestamp=1731369357691Num uniques 1149; 2024-11-11T23:55:57,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=9114, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-11T23:55:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731369357721, max timestamp=1731369357755Num uniques 1058; 2024-11-11T23:55:57,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-11T23:55:57,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731369357721, max timestamp=1731369357755Num uniques 1058; 2024-11-11T23:55:57,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-11T23:55:57,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731369357721, max timestamp=1731369357755Num uniques 1058; 2024-11-11T23:55:57,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-11T23:55:57,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/d4a8bb1c50e44329b67c49d1bcb7394a 2024-11-11T23:55:57,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/fd0e66b760df4420ac8a35a2820347ea is 50, key is test_row_0/B:col10/1731369357317/Put/seqid=0 2024-11-11T23:55:57,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742344_1520 (size=12001) 2024-11-11T23:55:57,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/fd0e66b760df4420ac8a35a2820347ea 2024-11-11T23:55:57,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/46e6d9089a084c558917c93903150835 is 50, key is test_row_0/C:col10/1731369357317/Put/seqid=0 2024-11-11T23:55:57,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742345_1521 (size=12001) 2024-11-11T23:55:57,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/46e6d9089a084c558917c93903150835 2024-11-11T23:55:57,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/d4a8bb1c50e44329b67c49d1bcb7394a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d4a8bb1c50e44329b67c49d1bcb7394a 2024-11-11T23:55:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731369357790, max timestamp=1731369357811Num uniques 1017; 2024-11-11T23:55:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-11T23:55:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731369357794, max timestamp=1731369357811Num uniques 1017; 2024-11-11T23:55:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731369357794, max timestamp=1731369357811Num uniques 1017; 2024-11-11T23:55:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=6870, suffix heap size=133736,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=150 2024-11-11T23:55:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b70d29849faf:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=6870, suffix heap size=133736,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=150 2024-11-11T23:55:57,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d4a8bb1c50e44329b67c49d1bcb7394a, entries=200, sequenceid=100, filesize=38.6 K 2024-11-11T23:55:57,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/fd0e66b760df4420ac8a35a2820347ea as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/fd0e66b760df4420ac8a35a2820347ea 2024-11-11T23:55:57,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/fd0e66b760df4420ac8a35a2820347ea, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T23:55:57,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/46e6d9089a084c558917c93903150835 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/46e6d9089a084c558917c93903150835 2024-11-11T23:55:57,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/46e6d9089a084c558917c93903150835, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T23:55:57,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=67.82 KB/69446 for 25afeec37b6a72df3a36de94b90eba91 in 507ms, sequenceid=100, compaction requested=true 2024-11-11T23:55:57,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:55:57,825 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:57,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:55:57,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:57,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:55:57,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:57,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:55:57,826 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:57,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:57,826 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:57,827 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/A is initiating minor compaction (all files) 2024-11-11T23:55:57,827 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/A in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:57,827 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:57,827 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/B is initiating minor compaction (all files) 2024-11-11T23:55:57,827 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/2beaf225d8244372a53caf2e7b45163b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d0a971a32aec4a7ebeae22050ad0199c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/54fef06bb67545319f0c3485bec616e6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d4a8bb1c50e44329b67c49d1bcb7394a] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=129.3 K 2024-11-11T23:55:57,827 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/B in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:57,827 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f3220f7c58b243efa22ec2d82e3b4c25, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/d87bc71472e241159709a572e49d339a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/eee6912e7c244fe9af80e325e9881bd8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/fd0e66b760df4420ac8a35a2820347ea] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=44.6 K 2024-11-11T23:55:57,827 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:57,827 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/2beaf225d8244372a53caf2e7b45163b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d0a971a32aec4a7ebeae22050ad0199c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/54fef06bb67545319f0c3485bec616e6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d4a8bb1c50e44329b67c49d1bcb7394a] 2024-11-11T23:55:57,827 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f3220f7c58b243efa22ec2d82e3b4c25, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731369353912 2024-11-11T23:55:57,827 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2beaf225d8244372a53caf2e7b45163b, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731369353912 2024-11-11T23:55:57,827 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting d87bc71472e241159709a572e49d339a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731369353976 2024-11-11T23:55:57,828 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting eee6912e7c244fe9af80e325e9881bd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731369354073 2024-11-11T23:55:57,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a971a32aec4a7ebeae22050ad0199c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731369353968 2024-11-11T23:55:57,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54fef06bb67545319f0c3485bec616e6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731369354073 2024-11-11T23:55:57,828 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0e66b760df4420ac8a35a2820347ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731369355154 2024-11-11T23:55:57,828 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a8bb1c50e44329b67c49d1bcb7394a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731369355154 2024-11-11T23:55:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:57,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=74.53 KB heapSize=206.56 KB 2024-11-11T23:55:57,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:55:57,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:57,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:55:57,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:57,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:55:57,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:55:57,837 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:57,838 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:57,839 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/1b53009cb22f4f38af41034edc336407 is 50, key is test_row_0/B:col10/1731369357317/Put/seqid=0 2024-11-11T23:55:57,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bf317e4cd3c44296b49d67617551e5f2_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369357822/Put/seqid=0 2024-11-11T23:55:57,841 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111bc169e40449c4510be954dca52af9ea6_25afeec37b6a72df3a36de94b90eba91 store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:57,844 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111bc169e40449c4510be954dca52af9ea6_25afeec37b6a72df3a36de94b90eba91, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:57,844 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bc169e40449c4510be954dca52af9ea6_25afeec37b6a72df3a36de94b90eba91 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:57,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742347_1523 (size=17534) 2024-11-11T23:55:57,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742348_1524 (size=4469) 2024-11-11T23:55:57,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:57,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742346_1522 (size=12139) 2024-11-11T23:55:57,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bf317e4cd3c44296b49d67617551e5f2_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf317e4cd3c44296b49d67617551e5f2_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:57,859 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/1b53009cb22f4f38af41034edc336407 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/1b53009cb22f4f38af41034edc336407 2024-11-11T23:55:57,859 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/965c85a7d441423a8eba8979c77d7fc2, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:57,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/965c85a7d441423a8eba8979c77d7fc2 is 175, key is test_row_0/A:col10/1731369357822/Put/seqid=0 2024-11-11T23:55:57,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742349_1525 (size=48639) 2024-11-11T23:55:57,864 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/B of 25afeec37b6a72df3a36de94b90eba91 into 1b53009cb22f4f38af41034edc336407(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:57,864 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:55:57,864 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/B, priority=12, startTime=1731369357826; duration=0sec 2024-11-11T23:55:57,864 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:55:57,864 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:B 2024-11-11T23:55:57,864 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:55:57,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369417857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369417856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,865 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:55:57,865 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/C is initiating minor compaction (all files) 2024-11-11T23:55:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,865 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/C in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,865 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2b87b943658446e2974dddf3a1d536ef, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/da3bb0a943804afbaf534cfc3f123099, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/81f6c9400d39421f84ac1d15df118b87, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/46e6d9089a084c558917c93903150835] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=44.6 K 2024-11-11T23:55:57,866 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b87b943658446e2974dddf3a1d536ef, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731369353912 2024-11-11T23:55:57,866 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting da3bb0a943804afbaf534cfc3f123099, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731369353976 2024-11-11T23:55:57,866 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f6c9400d39421f84ac1d15df118b87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731369354073 2024-11-11T23:55:57,866 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e6d9089a084c558917c93903150835, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731369355154 2024-11-11T23:55:57,871 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:57,872 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/faa4b1b2299741e78810c8c05df43425 is 50, key is test_row_0/C:col10/1731369357317/Put/seqid=0 2024-11-11T23:55:57,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369417865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369417865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742350_1526 (size=12139) 2024-11-11T23:55:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,878 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/faa4b1b2299741e78810c8c05df43425 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/faa4b1b2299741e78810c8c05df43425 2024-11-11T23:55:57,881 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/C of 25afeec37b6a72df3a36de94b90eba91 into faa4b1b2299741e78810c8c05df43425(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:57,881 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:55:57,881 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/C, priority=12, startTime=1731369357826; duration=0sec 2024-11-11T23:55:57,881 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:57,881 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:C 2024-11-11T23:55:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369417974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369417974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369417985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369417985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T23:55:58,065 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:55:58,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:55:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-11T23:55:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T23:55:58,066 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:55:58,067 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:55:58,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:55:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T23:55:58,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369418185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369418185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369418194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369418194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T23:55:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,255 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#A#compaction#451 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:55:58,256 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/e7e6bf8957704ef89a27eb9df0964f30 is 175, key is test_row_0/A:col10/1731369357317/Put/seqid=0 2024-11-11T23:55:58,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742351_1527 (size=31093) 2024-11-11T23:55:58,264 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/965c85a7d441423a8eba8979c77d7fc2 2024-11-11T23:55:58,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/733aa45390a748a6a58ac7fc0ac841e9 is 50, key is test_row_0/B:col10/1731369357822/Put/seqid=0 2024-11-11T23:55:58,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742352_1528 (size=12301) 2024-11-11T23:55:58,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369418275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,278 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:55:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T23:55:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T23:55:58,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369418494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369418495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369418505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369418505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:58,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T23:55:58,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:58,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,662 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/e7e6bf8957704ef89a27eb9df0964f30 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/e7e6bf8957704ef89a27eb9df0964f30 2024-11-11T23:55:58,666 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/A of 25afeec37b6a72df3a36de94b90eba91 into e7e6bf8957704ef89a27eb9df0964f30(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:55:58,666 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:55:58,666 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/A, priority=12, startTime=1731369357825; duration=0sec 2024-11-11T23:55:58,666 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:55:58,666 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:A 2024-11-11T23:55:58,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T23:55:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.08 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/733aa45390a748a6a58ac7fc0ac841e9 2024-11-11T23:55:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/d83235d71be64dcea1f9ead422dab848 is 50, key is test_row_0/C:col10/1731369357822/Put/seqid=0 2024-11-11T23:55:58,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742353_1529 (size=12301) 2024-11-11T23:55:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T23:55:58,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T23:55:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T23:55:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:55:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:58,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:59,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369419005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:59,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369419005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:59,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369419014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:59,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:55:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369419014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:55:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:55:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:55:59,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.08 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/d83235d71be64dcea1f9ead422dab848 2024-11-11T23:55:59,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/965c85a7d441423a8eba8979c77d7fc2 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/965c85a7d441423a8eba8979c77d7fc2 2024-11-11T23:55:59,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/965c85a7d441423a8eba8979c77d7fc2, entries=250, sequenceid=335, filesize=47.5 K 2024-11-11T23:55:59,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/733aa45390a748a6a58ac7fc0ac841e9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/733aa45390a748a6a58ac7fc0ac841e9 2024-11-11T23:55:59,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/733aa45390a748a6a58ac7fc0ac841e9, entries=150, sequenceid=335, filesize=12.0 K 2024-11-11T23:55:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/d83235d71be64dcea1f9ead422dab848 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/d83235d71be64dcea1f9ead422dab848 2024-11-11T23:55:59,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/d83235d71be64dcea1f9ead422dab848, entries=150, sequenceid=335, filesize=12.0 K 2024-11-11T23:55:59,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~81.24 KB/83186, heapSize ~226.44 KB/231872, currentSize=107.34 KB/109920 for 25afeec37b6a72df3a36de94b90eba91 in 1268ms, sequenceid=335, compaction requested=false 2024-11-11T23:55:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:55:59,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T23:55:59,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:55:59,134 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T23:55:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:55:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:55:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:55:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:55:59,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154160292d4ab4ac3bb9d086e0b4f1c89_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369357855/Put/seqid=0 2024-11-11T23:55:59,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742354_1530 (size=12454) 2024-11-11T23:55:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T23:55:59,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:59,545 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154160292d4ab4ac3bb9d086e0b4f1c89_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154160292d4ab4ac3bb9d086e0b4f1c89_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:55:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/ff91f1b7f4ff4d3181b5febcb221edc6, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:55:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/ff91f1b7f4ff4d3181b5febcb221edc6 is 175, key is test_row_0/A:col10/1731369357855/Put/seqid=0 2024-11-11T23:55:59,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742355_1531 (size=31255) 2024-11-11T23:55:59,950 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/ff91f1b7f4ff4d3181b5febcb221edc6 2024-11-11T23:55:59,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/f89f58a3f58846878a6da646eae8a795 is 50, key is test_row_0/B:col10/1731369357855/Put/seqid=0 2024-11-11T23:55:59,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742356_1532 (size=12301) 2024-11-11T23:56:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:00,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:00,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369420034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369420038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369420039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369420039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369420145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369420154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369420154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369420154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T23:56:00,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369420354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,359 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/f89f58a3f58846878a6da646eae8a795 2024-11-11T23:56:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/c8f9bedaeca945b29e1dbee05abe00fc is 50, key is test_row_0/C:col10/1731369357855/Put/seqid=0 2024-11-11T23:56:00,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742357_1533 (size=12301) 2024-11-11T23:56:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369420364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369420364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369420364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369420665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369420674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369420674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369420674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:00,768 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/c8f9bedaeca945b29e1dbee05abe00fc 2024-11-11T23:56:00,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/ff91f1b7f4ff4d3181b5febcb221edc6 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/ff91f1b7f4ff4d3181b5febcb221edc6 2024-11-11T23:56:00,774 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/ff91f1b7f4ff4d3181b5febcb221edc6, entries=150, sequenceid=357, filesize=30.5 K 2024-11-11T23:56:00,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/f89f58a3f58846878a6da646eae8a795 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f89f58a3f58846878a6da646eae8a795 2024-11-11T23:56:00,778 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f89f58a3f58846878a6da646eae8a795, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T23:56:00,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/c8f9bedaeca945b29e1dbee05abe00fc as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/c8f9bedaeca945b29e1dbee05abe00fc 2024-11-11T23:56:00,781 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/c8f9bedaeca945b29e1dbee05abe00fc, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T23:56:00,782 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 25afeec37b6a72df3a36de94b90eba91 in 1648ms, sequenceid=357, compaction requested=true 2024-11-11T23:56:00,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:00,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:00,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-11T23:56:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-11T23:56:00,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-11T23:56:00,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7160 sec 2024-11-11T23:56:00,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.7200 sec 2024-11-11T23:56:01,036 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T23:56:01,036 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T23:56:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:01,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T23:56:01,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:01,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:01,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:01,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:01,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:01,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:01,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117b666caf68b041fca0b29ace0a14a482_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369360038/Put/seqid=0 2024-11-11T23:56:01,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742358_1534 (size=12454) 2024-11-11T23:56:01,188 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:01,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117b666caf68b041fca0b29ace0a14a482_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117b666caf68b041fca0b29ace0a14a482_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:01,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/28c84d802d0e475391d516a872f8f3f9, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:01,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/28c84d802d0e475391d516a872f8f3f9 is 175, key is test_row_0/A:col10/1731369360038/Put/seqid=0 2024-11-11T23:56:01,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742359_1535 (size=31255) 2024-11-11T23:56:01,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369421198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369421200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369421199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369421201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369421314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369421314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369421314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369421314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369421524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369421524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369421524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369421524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/28c84d802d0e475391d516a872f8f3f9 2024-11-11T23:56:01,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/bd39fa31d7494fb8819c7d576c2107af is 50, key is test_row_0/B:col10/1731369360038/Put/seqid=0 2024-11-11T23:56:01,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742360_1536 (size=12301) 2024-11-11T23:56:01,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369421834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369421834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369421834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369421835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/bd39fa31d7494fb8819c7d576c2107af 2024-11-11T23:56:02,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/4581d08da4084754a1c0075435499e79 is 50, key is test_row_0/C:col10/1731369360038/Put/seqid=0 2024-11-11T23:56:02,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742361_1537 (size=12301) 2024-11-11T23:56:02,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/4581d08da4084754a1c0075435499e79 2024-11-11T23:56:02,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/28c84d802d0e475391d516a872f8f3f9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/28c84d802d0e475391d516a872f8f3f9 2024-11-11T23:56:02,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/28c84d802d0e475391d516a872f8f3f9, entries=150, sequenceid=372, filesize=30.5 K 2024-11-11T23:56:02,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/bd39fa31d7494fb8819c7d576c2107af as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/bd39fa31d7494fb8819c7d576c2107af 2024-11-11T23:56:02,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/bd39fa31d7494fb8819c7d576c2107af, entries=150, sequenceid=372, filesize=12.0 K 2024-11-11T23:56:02,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/4581d08da4084754a1c0075435499e79 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4581d08da4084754a1c0075435499e79 2024-11-11T23:56:02,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4581d08da4084754a1c0075435499e79, entries=150, sequenceid=372, filesize=12.0 K 2024-11-11T23:56:02,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 25afeec37b6a72df3a36de94b90eba91 in 846ms, sequenceid=372, compaction requested=true 2024-11-11T23:56:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:56:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:02,025 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:56:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:56:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:02,025 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:56:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:56:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:02,026 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:56:02,026 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:56:02,026 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/A is initiating minor compaction (all files) 2024-11-11T23:56:02,026 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/B is initiating minor compaction (all files) 2024-11-11T23:56:02,026 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/A in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,026 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/B in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,026 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/e7e6bf8957704ef89a27eb9df0964f30, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/965c85a7d441423a8eba8979c77d7fc2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/ff91f1b7f4ff4d3181b5febcb221edc6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/28c84d802d0e475391d516a872f8f3f9] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=138.9 K 2024-11-11T23:56:02,026 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/1b53009cb22f4f38af41034edc336407, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/733aa45390a748a6a58ac7fc0ac841e9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f89f58a3f58846878a6da646eae8a795, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/bd39fa31d7494fb8819c7d576c2107af] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=47.9 K 2024-11-11T23:56:02,027 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/e7e6bf8957704ef89a27eb9df0964f30, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/965c85a7d441423a8eba8979c77d7fc2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/ff91f1b7f4ff4d3181b5febcb221edc6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/28c84d802d0e475391d516a872f8f3f9] 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b53009cb22f4f38af41034edc336407, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731369355154 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7e6bf8957704ef89a27eb9df0964f30, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731369355154 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 733aa45390a748a6a58ac7fc0ac841e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731369357816 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 965c85a7d441423a8eba8979c77d7fc2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731369357816 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f89f58a3f58846878a6da646eae8a795, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369357851 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff91f1b7f4ff4d3181b5febcb221edc6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369357851 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting bd39fa31d7494fb8819c7d576c2107af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731369360038 2024-11-11T23:56:02,027 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28c84d802d0e475391d516a872f8f3f9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731369360038 2024-11-11T23:56:02,032 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:02,035 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#B#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:02,035 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/e41a2678b3714a3b976df0a80d9ddc6b is 50, key is test_row_0/B:col10/1731369360038/Put/seqid=0 2024-11-11T23:56:02,038 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112f78bc5d70e2481d882e75c3a6de9ab2_25afeec37b6a72df3a36de94b90eba91 store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:02,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742362_1538 (size=12575) 2024-11-11T23:56:02,041 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112f78bc5d70e2481d882e75c3a6de9ab2_25afeec37b6a72df3a36de94b90eba91, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:02,041 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112f78bc5d70e2481d882e75c3a6de9ab2_25afeec37b6a72df3a36de94b90eba91 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:02,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742363_1539 (size=4469) 2024-11-11T23:56:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T23:56:02,225 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:56:02,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:56:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-11T23:56:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T23:56:02,226 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:56:02,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:56:02,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:56:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:02,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T23:56:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:02,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:02,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118991b580b85f407398030ab6df467128_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369361185/Put/seqid=0 2024-11-11T23:56:02,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742364_1540 (size=14994) 2024-11-11T23:56:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T23:56:02,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369422345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369422351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369422352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369422352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369422353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,443 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/e41a2678b3714a3b976df0a80d9ddc6b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e41a2678b3714a3b976df0a80d9ddc6b 2024-11-11T23:56:02,446 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/B of 25afeec37b6a72df3a36de94b90eba91 into e41a2678b3714a3b976df0a80d9ddc6b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:02,446 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:02,446 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:02,446 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/B, priority=12, startTime=1731369362025; duration=0sec 2024-11-11T23:56:02,446 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:02,446 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:B 2024-11-11T23:56:02,446 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:56:02,447 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/050b36beaefc42e38b5fc0c24c4c4a69 is 175, key is test_row_0/A:col10/1731369360038/Put/seqid=0 2024-11-11T23:56:02,448 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:56:02,448 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/C is initiating minor compaction (all files) 2024-11-11T23:56:02,448 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/C in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,448 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/faa4b1b2299741e78810c8c05df43425, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/d83235d71be64dcea1f9ead422dab848, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/c8f9bedaeca945b29e1dbee05abe00fc, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4581d08da4084754a1c0075435499e79] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=47.9 K 2024-11-11T23:56:02,449 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting faa4b1b2299741e78810c8c05df43425, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731369355154 2024-11-11T23:56:02,449 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting d83235d71be64dcea1f9ead422dab848, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731369357816 2024-11-11T23:56:02,449 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f9bedaeca945b29e1dbee05abe00fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731369357851 2024-11-11T23:56:02,449 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4581d08da4084754a1c0075435499e79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731369360038 2024-11-11T23:56:02,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742365_1541 (size=31529) 2024-11-11T23:56:02,462 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/050b36beaefc42e38b5fc0c24c4c4a69 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/050b36beaefc42e38b5fc0c24c4c4a69 2024-11-11T23:56:02,466 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#C#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:02,466 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/A of 25afeec37b6a72df3a36de94b90eba91 into 050b36beaefc42e38b5fc0c24c4c4a69(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:02,466 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:02,466 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/A, priority=12, startTime=1731369362025; duration=0sec 2024-11-11T23:56:02,466 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:02,467 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:A 2024-11-11T23:56:02,467 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/abb75a9f0d374da6b7789cea26d8b4f5 is 50, key is test_row_0/C:col10/1731369360038/Put/seqid=0 2024-11-11T23:56:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369422464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369422464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369422464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369422465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742366_1542 (size=12575) 2024-11-11T23:56:02,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T23:56:02,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369422674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369422674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369422674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369422674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:02,714 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118991b580b85f407398030ab6df467128_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118991b580b85f407398030ab6df467128_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:02,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/bb0fbe935a224913a48ae895ca38c0cd, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:02,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/bb0fbe935a224913a48ae895ca38c0cd is 175, key is test_row_0/A:col10/1731369361185/Put/seqid=0 2024-11-11T23:56:02,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742367_1543 (size=39949) 2024-11-11T23:56:02,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T23:56:02,881 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/abb75a9f0d374da6b7789cea26d8b4f5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/abb75a9f0d374da6b7789cea26d8b4f5 2024-11-11T23:56:02,884 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/C of 25afeec37b6a72df3a36de94b90eba91 into abb75a9f0d374da6b7789cea26d8b4f5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:02,884 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:02,884 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/C, priority=12, startTime=1731369362025; duration=0sec 2024-11-11T23:56:02,884 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:02,884 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:C 2024-11-11T23:56:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369422985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369422985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369422985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369422986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:02,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:02,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=390, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/bb0fbe935a224913a48ae895ca38c0cd 2024-11-11T23:56:03,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/857f886d1f3e4bc4a4a7a7e6437369f6 is 50, key is test_row_0/B:col10/1731369361185/Put/seqid=0 2024-11-11T23:56:03,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742368_1544 (size=12301) 2024-11-11T23:56:03,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:03,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:03,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T23:56:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369423364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:03,366 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:03,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:03,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:03,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369423494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:03,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369423494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:03,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369423495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369423495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:03,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/857f886d1f3e4bc4a4a7a7e6437369f6 2024-11-11T23:56:03,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/8daafab018ba44eaa4d4ba8723e70854 is 50, key is test_row_0/C:col10/1731369361185/Put/seqid=0 2024-11-11T23:56:03,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742369_1545 (size=12301) 2024-11-11T23:56:03,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:03,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:03,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:03,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:03,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:03,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:03,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:03,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:03,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/8daafab018ba44eaa4d4ba8723e70854 2024-11-11T23:56:03,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/bb0fbe935a224913a48ae895ca38c0cd as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/bb0fbe935a224913a48ae895ca38c0cd 2024-11-11T23:56:03,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/bb0fbe935a224913a48ae895ca38c0cd, entries=200, sequenceid=390, filesize=39.0 K 2024-11-11T23:56:03,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/857f886d1f3e4bc4a4a7a7e6437369f6 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/857f886d1f3e4bc4a4a7a7e6437369f6 2024-11-11T23:56:03,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/857f886d1f3e4bc4a4a7a7e6437369f6, entries=150, sequenceid=390, filesize=12.0 K 2024-11-11T23:56:03,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/8daafab018ba44eaa4d4ba8723e70854 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8daafab018ba44eaa4d4ba8723e70854 2024-11-11T23:56:03,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8daafab018ba44eaa4d4ba8723e70854, entries=150, sequenceid=390, filesize=12.0 K 2024-11-11T23:56:03,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 25afeec37b6a72df3a36de94b90eba91 in 1651ms, sequenceid=390, compaction requested=false 2024-11-11T23:56:03,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:04,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T23:56:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:04,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T23:56:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0f88ba102414d869ace2a2e97691df0_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369362351/Put/seqid=0 2024-11-11T23:56:04,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742370_1546 (size=12454) 2024-11-11T23:56:04,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:04,072 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0f88ba102414d869ace2a2e97691df0_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0f88ba102414d869ace2a2e97691df0_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:04,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/3e0c8e7241094c0b9f2b7f46401b1697, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:04,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/3e0c8e7241094c0b9f2b7f46401b1697 is 175, key is test_row_0/A:col10/1731369362351/Put/seqid=0 2024-11-11T23:56:04,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742371_1547 (size=31255) 2024-11-11T23:56:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T23:56:04,476 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/3e0c8e7241094c0b9f2b7f46401b1697 2024-11-11T23:56:04,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/e39b0276ab2a49d5ac43b66b8d184ea0 is 50, key is test_row_0/B:col10/1731369362351/Put/seqid=0 2024-11-11T23:56:04,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742372_1548 (size=12301) 2024-11-11T23:56:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:04,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:04,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369424530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369424530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369424531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369424532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369424644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369424644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369424644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369424644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369424854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369424854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369424854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369424855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:04,884 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/e39b0276ab2a49d5ac43b66b8d184ea0 2024-11-11T23:56:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/2fef02c4a7dd4a7ab1ef9600ae2fea06 is 50, key is test_row_0/C:col10/1731369362351/Put/seqid=0 2024-11-11T23:56:04,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742373_1549 (size=12301) 2024-11-11T23:56:05,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369425165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369425165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369425165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369425165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,293 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/2fef02c4a7dd4a7ab1ef9600ae2fea06 2024-11-11T23:56:05,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/3e0c8e7241094c0b9f2b7f46401b1697 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3e0c8e7241094c0b9f2b7f46401b1697 2024-11-11T23:56:05,302 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3e0c8e7241094c0b9f2b7f46401b1697, entries=150, sequenceid=408, filesize=30.5 K 2024-11-11T23:56:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/e39b0276ab2a49d5ac43b66b8d184ea0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e39b0276ab2a49d5ac43b66b8d184ea0 2024-11-11T23:56:05,305 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e39b0276ab2a49d5ac43b66b8d184ea0, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T23:56:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/2fef02c4a7dd4a7ab1ef9600ae2fea06 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2fef02c4a7dd4a7ab1ef9600ae2fea06 2024-11-11T23:56:05,308 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2fef02c4a7dd4a7ab1ef9600ae2fea06, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T23:56:05,309 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 25afeec37b6a72df3a36de94b90eba91 in 1248ms, sequenceid=408, compaction requested=true 2024-11-11T23:56:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-11T23:56:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-11T23:56:05,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T23:56:05,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0830 sec 2024-11-11T23:56:05,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.0870 sec 2024-11-11T23:56:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:05,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T23:56:05,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:05,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:05,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:05,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:05,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:05,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:05,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116a0a38d2de7b44d0b4fa9523aa49daf3_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369364523/Put/seqid=0 2024-11-11T23:56:05,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742374_1550 (size=14994) 2024-11-11T23:56:05,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369425450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369425564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369425674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369425674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369425674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369425674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369425774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:05,806 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:05,809 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116a0a38d2de7b44d0b4fa9523aa49daf3_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116a0a38d2de7b44d0b4fa9523aa49daf3_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:05,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/7b5fc4945fc44782abbec7d095653d51, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:05,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/7b5fc4945fc44782abbec7d095653d51 is 175, key is test_row_0/A:col10/1731369364523/Put/seqid=0 2024-11-11T23:56:05,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742375_1551 (size=39949) 2024-11-11T23:56:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369426085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:06,213 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/7b5fc4945fc44782abbec7d095653d51 2024-11-11T23:56:06,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/cbb28a374a314ad38511bcf1b599f2c3 is 50, key is test_row_0/B:col10/1731369364523/Put/seqid=0 2024-11-11T23:56:06,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742376_1552 (size=12301) 2024-11-11T23:56:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T23:56:06,394 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:56:06,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:56:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-11T23:56:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T23:56:06,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:56:06,397 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:56:06,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:56:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T23:56:06,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T23:56:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:06,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:06,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369426594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:06,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:06,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/cbb28a374a314ad38511bcf1b599f2c3 2024-11-11T23:56:06,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/f2e43be3ff834a369e8f5636792b0fc3 is 50, key is test_row_0/C:col10/1731369364523/Put/seqid=0 2024-11-11T23:56:06,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742377_1553 (size=12301) 2024-11-11T23:56:06,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369426684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:06,687 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:06,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369426694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369426694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369426695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:06,697 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:06,697 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,697 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:06,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T23:56:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:06,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:06,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T23:56:06,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T23:56:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:06,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:07,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T23:56:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:07,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T23:56:07,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/f2e43be3ff834a369e8f5636792b0fc3 2024-11-11T23:56:07,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/7b5fc4945fc44782abbec7d095653d51 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/7b5fc4945fc44782abbec7d095653d51 2024-11-11T23:56:07,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/7b5fc4945fc44782abbec7d095653d51, entries=200, sequenceid=426, filesize=39.0 K 2024-11-11T23:56:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/cbb28a374a314ad38511bcf1b599f2c3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/cbb28a374a314ad38511bcf1b599f2c3 2024-11-11T23:56:07,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/cbb28a374a314ad38511bcf1b599f2c3, entries=150, sequenceid=426, filesize=12.0 K 2024-11-11T23:56:07,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/f2e43be3ff834a369e8f5636792b0fc3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/f2e43be3ff834a369e8f5636792b0fc3 2024-11-11T23:56:07,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/f2e43be3ff834a369e8f5636792b0fc3, entries=150, sequenceid=426, filesize=12.0 K 2024-11-11T23:56:07,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 25afeec37b6a72df3a36de94b90eba91 in 1647ms, sequenceid=426, compaction requested=true 2024-11-11T23:56:07,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:07,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:56:07,043 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:56:07,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:07,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:56:07,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:07,043 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:56:07,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:56:07,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:07,044 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:56:07,044 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49478 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:56:07,044 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/B is initiating minor compaction (all files) 2024-11-11T23:56:07,044 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/A is initiating minor compaction (all files) 2024-11-11T23:56:07,044 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/A in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:07,044 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/B in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:07,044 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/050b36beaefc42e38b5fc0c24c4c4a69, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/bb0fbe935a224913a48ae895ca38c0cd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3e0c8e7241094c0b9f2b7f46401b1697, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/7b5fc4945fc44782abbec7d095653d51] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=139.3 K 2024-11-11T23:56:07,044 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e41a2678b3714a3b976df0a80d9ddc6b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/857f886d1f3e4bc4a4a7a7e6437369f6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e39b0276ab2a49d5ac43b66b8d184ea0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/cbb28a374a314ad38511bcf1b599f2c3] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=48.3 K 2024-11-11T23:56:07,044 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:07,044 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/050b36beaefc42e38b5fc0c24c4c4a69, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/bb0fbe935a224913a48ae895ca38c0cd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3e0c8e7241094c0b9f2b7f46401b1697, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/7b5fc4945fc44782abbec7d095653d51] 2024-11-11T23:56:07,044 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting e41a2678b3714a3b976df0a80d9ddc6b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731369360038 2024-11-11T23:56:07,045 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050b36beaefc42e38b5fc0c24c4c4a69, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731369360038 2024-11-11T23:56:07,045 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 857f886d1f3e4bc4a4a7a7e6437369f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369361185 2024-11-11T23:56:07,045 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb0fbe935a224913a48ae895ca38c0cd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369361185 2024-11-11T23:56:07,045 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting e39b0276ab2a49d5ac43b66b8d184ea0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731369362324 2024-11-11T23:56:07,045 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e0c8e7241094c0b9f2b7f46401b1697, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731369362324 2024-11-11T23:56:07,045 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting cbb28a374a314ad38511bcf1b599f2c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731369364523 2024-11-11T23:56:07,046 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5fc4945fc44782abbec7d095653d51, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731369364523 2024-11-11T23:56:07,065 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:07,067 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c96311f596e545dcb10e869b2e2add2f_25afeec37b6a72df3a36de94b90eba91 store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:07,067 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#B#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:07,068 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/8f1dca878cbd42d0a0100e4ae67170c8 is 50, key is test_row_0/B:col10/1731369364523/Put/seqid=0 2024-11-11T23:56:07,068 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c96311f596e545dcb10e869b2e2add2f_25afeec37b6a72df3a36de94b90eba91, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:07,069 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c96311f596e545dcb10e869b2e2add2f_25afeec37b6a72df3a36de94b90eba91 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:07,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742379_1555 (size=4469) 2024-11-11T23:56:07,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742378_1554 (size=12711) 2024-11-11T23:56:07,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T23:56:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:07,159 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T23:56:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:07,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:07,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:07,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ea2e1f3389024503b95ebe23c3a7e868_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369365449/Put/seqid=0 2024-11-11T23:56:07,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742380_1556 (size=12454) 2024-11-11T23:56:07,479 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:07,480 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/453eef3f409a4bb7bbc1608943646d56 is 175, key is test_row_0/A:col10/1731369364523/Put/seqid=0 2024-11-11T23:56:07,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742381_1557 (size=31665) 2024-11-11T23:56:07,490 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/8f1dca878cbd42d0a0100e4ae67170c8 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8f1dca878cbd42d0a0100e4ae67170c8 2024-11-11T23:56:07,493 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/B of 25afeec37b6a72df3a36de94b90eba91 into 8f1dca878cbd42d0a0100e4ae67170c8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:07,493 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:07,493 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/B, priority=12, startTime=1731369367043; duration=0sec 2024-11-11T23:56:07,493 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:07,494 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:B 2024-11-11T23:56:07,494 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T23:56:07,495 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49478 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T23:56:07,495 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/C is initiating minor compaction (all files) 2024-11-11T23:56:07,495 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/C in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:07,495 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/abb75a9f0d374da6b7789cea26d8b4f5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8daafab018ba44eaa4d4ba8723e70854, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2fef02c4a7dd4a7ab1ef9600ae2fea06, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/f2e43be3ff834a369e8f5636792b0fc3] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=48.3 K 2024-11-11T23:56:07,495 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting abb75a9f0d374da6b7789cea26d8b4f5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731369360038 2024-11-11T23:56:07,495 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8daafab018ba44eaa4d4ba8723e70854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731369361185 2024-11-11T23:56:07,496 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fef02c4a7dd4a7ab1ef9600ae2fea06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731369362324 2024-11-11T23:56:07,496 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e43be3ff834a369e8f5636792b0fc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731369364523 2024-11-11T23:56:07,504 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:07,504 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/8707b8c5fbc54fd8b6605e83534e8bcb is 50, key is test_row_0/C:col10/1731369364523/Put/seqid=0 2024-11-11T23:56:07,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742382_1558 (size=12711) 2024-11-11T23:56:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T23:56:07,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:07,571 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ea2e1f3389024503b95ebe23c3a7e868_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ea2e1f3389024503b95ebe23c3a7e868_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:07,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/8e1eb7a8ac6f4dcb8b49905b03615c63, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:07,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/8e1eb7a8ac6f4dcb8b49905b03615c63 is 175, key is test_row_0/A:col10/1731369365449/Put/seqid=0 2024-11-11T23:56:07,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742383_1559 (size=31255) 2024-11-11T23:56:07,575 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=440, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/8e1eb7a8ac6f4dcb8b49905b03615c63 2024-11-11T23:56:07,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/febf8b984aa8442da2199bcacaa7e270 is 50, key is test_row_0/B:col10/1731369365449/Put/seqid=0 2024-11-11T23:56:07,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742384_1560 (size=12301) 2024-11-11T23:56:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:07,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:07,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369427685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:07,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369427794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:07,886 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/453eef3f409a4bb7bbc1608943646d56 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/453eef3f409a4bb7bbc1608943646d56 2024-11-11T23:56:07,890 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/A of 25afeec37b6a72df3a36de94b90eba91 into 453eef3f409a4bb7bbc1608943646d56(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:07,890 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:07,890 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/A, priority=12, startTime=1731369367043; duration=0sec 2024-11-11T23:56:07,890 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:07,890 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:A 2024-11-11T23:56:07,927 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/8707b8c5fbc54fd8b6605e83534e8bcb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8707b8c5fbc54fd8b6605e83534e8bcb 2024-11-11T23:56:07,930 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/C of 25afeec37b6a72df3a36de94b90eba91 into 8707b8c5fbc54fd8b6605e83534e8bcb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:07,930 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:07,931 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/C, priority=12, startTime=1731369367043; duration=0sec 2024-11-11T23:56:07,931 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:07,931 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:C 2024-11-11T23:56:07,985 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/febf8b984aa8442da2199bcacaa7e270 2024-11-11T23:56:07,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/88d19d523601491ea4c44082bddf0e57 is 50, key is test_row_0/C:col10/1731369365449/Put/seqid=0 2024-11-11T23:56:07,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742385_1561 (size=12301) 2024-11-11T23:56:07,994 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/88d19d523601491ea4c44082bddf0e57 2024-11-11T23:56:07,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/8e1eb7a8ac6f4dcb8b49905b03615c63 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8e1eb7a8ac6f4dcb8b49905b03615c63 2024-11-11T23:56:08,002 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8e1eb7a8ac6f4dcb8b49905b03615c63, entries=150, sequenceid=440, filesize=30.5 K 2024-11-11T23:56:08,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/febf8b984aa8442da2199bcacaa7e270 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/febf8b984aa8442da2199bcacaa7e270 2024-11-11T23:56:08,006 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/febf8b984aa8442da2199bcacaa7e270, entries=150, sequenceid=440, filesize=12.0 K 2024-11-11T23:56:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/88d19d523601491ea4c44082bddf0e57 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/88d19d523601491ea4c44082bddf0e57 2024-11-11T23:56:08,009 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/88d19d523601491ea4c44082bddf0e57, entries=150, sequenceid=440, filesize=12.0 K 2024-11-11T23:56:08,010 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 25afeec37b6a72df3a36de94b90eba91 in 850ms, sequenceid=440, compaction requested=false 2024-11-11T23:56:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T23:56:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:08,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T23:56:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-11T23:56:08,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:08,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:08,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:08,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:08,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:08,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:08,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T23:56:08,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-11T23:56:08,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.6180 sec 2024-11-11T23:56:08,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bbe972d2aa96404c81a59e296068eca7_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369368009/Put/seqid=0 2024-11-11T23:56:08,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742386_1562 (size=14994) 2024-11-11T23:56:08,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369428053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:08,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369428164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369428374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:08,422 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:08,425 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bbe972d2aa96404c81a59e296068eca7_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bbe972d2aa96404c81a59e296068eca7_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:08,425 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/50f3aa8294e84e29b3867900aa624065, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:08,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/50f3aa8294e84e29b3867900aa624065 is 175, key is test_row_0/A:col10/1731369368009/Put/seqid=0 2024-11-11T23:56:08,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742387_1563 (size=39949) 2024-11-11T23:56:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T23:56:08,545 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:56:08,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:56:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-11T23:56:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T23:56:08,547 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:56:08,547 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:56:08,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:56:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T23:56:08,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369428685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:08,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369428694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:08,695 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:08,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T23:56:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:08,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:08,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369428704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:08,707 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369428724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369428724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:08,726 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,726 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:08,840 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=462, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/50f3aa8294e84e29b3867900aa624065 2024-11-11T23:56:08,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/ae327435777d4cc887303d5d8793bf8c is 50, key is test_row_0/B:col10/1731369368009/Put/seqid=0 2024-11-11T23:56:08,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742388_1564 (size=12301) 2024-11-11T23:56:08,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T23:56:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:08,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T23:56:09,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T23:56:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:09,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T23:56:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T23:56:09,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369429194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:09,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/ae327435777d4cc887303d5d8793bf8c 2024-11-11T23:56:09,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/4b395b7c1f384b0e989be9e66b5e67e0 is 50, key is test_row_0/C:col10/1731369368009/Put/seqid=0 2024-11-11T23:56:09,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742389_1565 (size=12301) 2024-11-11T23:56:09,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/4b395b7c1f384b0e989be9e66b5e67e0 2024-11-11T23:56:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/50f3aa8294e84e29b3867900aa624065 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/50f3aa8294e84e29b3867900aa624065 2024-11-11T23:56:09,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/50f3aa8294e84e29b3867900aa624065, entries=200, sequenceid=462, filesize=39.0 K 2024-11-11T23:56:09,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/ae327435777d4cc887303d5d8793bf8c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/ae327435777d4cc887303d5d8793bf8c 2024-11-11T23:56:09,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/ae327435777d4cc887303d5d8793bf8c, entries=150, sequenceid=462, filesize=12.0 K 2024-11-11T23:56:09,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/4b395b7c1f384b0e989be9e66b5e67e0 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4b395b7c1f384b0e989be9e66b5e67e0 2024-11-11T23:56:09,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4b395b7c1f384b0e989be9e66b5e67e0, entries=150, sequenceid=462, filesize=12.0 K 2024-11-11T23:56:09,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 25afeec37b6a72df3a36de94b90eba91 in 1261ms, sequenceid=462, compaction requested=true 2024-11-11T23:56:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:56:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:09,271 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:56:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:09,271 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:56:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:09,272 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:09,272 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:09,272 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/B is initiating minor compaction (all files) 2024-11-11T23:56:09,272 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/A is initiating minor compaction (all files) 2024-11-11T23:56:09,272 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/B in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,272 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/A in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,272 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/453eef3f409a4bb7bbc1608943646d56, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8e1eb7a8ac6f4dcb8b49905b03615c63, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/50f3aa8294e84e29b3867900aa624065] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=100.5 K 2024-11-11T23:56:09,272 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8f1dca878cbd42d0a0100e4ae67170c8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/febf8b984aa8442da2199bcacaa7e270, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/ae327435777d4cc887303d5d8793bf8c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=36.4 K 2024-11-11T23:56:09,272 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,272 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/453eef3f409a4bb7bbc1608943646d56, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8e1eb7a8ac6f4dcb8b49905b03615c63, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/50f3aa8294e84e29b3867900aa624065] 2024-11-11T23:56:09,273 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1dca878cbd42d0a0100e4ae67170c8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731369364523 2024-11-11T23:56:09,273 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 453eef3f409a4bb7bbc1608943646d56, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731369364523 2024-11-11T23:56:09,273 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting febf8b984aa8442da2199bcacaa7e270, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731369365440 2024-11-11T23:56:09,273 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e1eb7a8ac6f4dcb8b49905b03615c63, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731369365440 2024-11-11T23:56:09,273 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50f3aa8294e84e29b3867900aa624065, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731369367636 2024-11-11T23:56:09,273 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting ae327435777d4cc887303d5d8793bf8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731369367636 2024-11-11T23:56:09,279 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:09,281 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111173c0b1c9599347128bc3a25060e0c14d_25afeec37b6a72df3a36de94b90eba91 store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:09,282 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:09,282 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/02026b67d8764964a5362971b25479a7 is 50, key is test_row_0/B:col10/1731369368009/Put/seqid=0 2024-11-11T23:56:09,283 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111173c0b1c9599347128bc3a25060e0c14d_25afeec37b6a72df3a36de94b90eba91, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:09,283 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111173c0b1c9599347128bc3a25060e0c14d_25afeec37b6a72df3a36de94b90eba91 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:09,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742390_1566 (size=4469) 2024-11-11T23:56:09,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T23:56:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,309 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T23:56:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:09,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742391_1567 (size=12813) 2024-11-11T23:56:09,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118357a69431274319bb289d929de4564d_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369368048/Put/seqid=0 2024-11-11T23:56:09,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742392_1568 (size=12454) 2024-11-11T23:56:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T23:56:09,704 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:09,705 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/8b2dd23549c24afca819bd6cfb293075 is 175, key is test_row_0/A:col10/1731369368009/Put/seqid=0 2024-11-11T23:56:09,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742393_1569 (size=31767) 2024-11-11T23:56:09,710 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/8b2dd23549c24afca819bd6cfb293075 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8b2dd23549c24afca819bd6cfb293075 2024-11-11T23:56:09,714 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/A of 25afeec37b6a72df3a36de94b90eba91 into 8b2dd23549c24afca819bd6cfb293075(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:09,714 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:09,714 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/A, priority=13, startTime=1731369369271; duration=0sec 2024-11-11T23:56:09,714 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:09,714 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:A 2024-11-11T23:56:09,715 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:09,715 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:09,716 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/C is initiating minor compaction (all files) 2024-11-11T23:56:09,716 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/C in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:09,716 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8707b8c5fbc54fd8b6605e83534e8bcb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/88d19d523601491ea4c44082bddf0e57, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4b395b7c1f384b0e989be9e66b5e67e0] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=36.4 K 2024-11-11T23:56:09,716 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8707b8c5fbc54fd8b6605e83534e8bcb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731369364523 2024-11-11T23:56:09,716 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d19d523601491ea4c44082bddf0e57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731369365440 2024-11-11T23:56:09,717 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b395b7c1f384b0e989be9e66b5e67e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731369367636 2024-11-11T23:56:09,717 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/02026b67d8764964a5362971b25479a7 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/02026b67d8764964a5362971b25479a7 2024-11-11T23:56:09,721 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/B of 25afeec37b6a72df3a36de94b90eba91 into 02026b67d8764964a5362971b25479a7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:09,721 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:09,721 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/B, priority=13, startTime=1731369369271; duration=0sec 2024-11-11T23:56:09,721 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:09,721 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:B 2024-11-11T23:56:09,722 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:09,722 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/8613b207880540c7a81eaefd4ba8cf25 is 50, key is test_row_0/C:col10/1731369368009/Put/seqid=0 2024-11-11T23:56:09,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742394_1570 (size=12813) 2024-11-11T23:56:09,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:09,742 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118357a69431274319bb289d929de4564d_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118357a69431274319bb289d929de4564d_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:09,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee is 175, key is test_row_0/A:col10/1731369368048/Put/seqid=0 2024-11-11T23:56:09,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742395_1571 (size=31255) 2024-11-11T23:56:10,129 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/8613b207880540c7a81eaefd4ba8cf25 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8613b207880540c7a81eaefd4ba8cf25 2024-11-11T23:56:10,132 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/C of 25afeec37b6a72df3a36de94b90eba91 into 8613b207880540c7a81eaefd4ba8cf25(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:10,132 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:10,132 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/C, priority=13, startTime=1731369369271; duration=0sec 2024-11-11T23:56:10,132 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:10,132 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:C 2024-11-11T23:56:10,146 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=475, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee 2024-11-11T23:56:10,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/acb11c68d19444bcac569523211b7ec1 is 50, key is test_row_0/B:col10/1731369368048/Put/seqid=0 2024-11-11T23:56:10,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742396_1572 (size=12301) 2024-11-11T23:56:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:10,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:10,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369430264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:10,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369430374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:10,556 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/acb11c68d19444bcac569523211b7ec1 2024-11-11T23:56:10,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/47f6ba4a0e9144f3890b186b6380d4aa is 50, key is test_row_0/C:col10/1731369368048/Put/seqid=0 2024-11-11T23:56:10,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742397_1573 (size=12301) 2024-11-11T23:56:10,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369430584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T23:56:10,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369430894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:10,964 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/47f6ba4a0e9144f3890b186b6380d4aa 2024-11-11T23:56:10,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee 2024-11-11T23:56:10,970 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee, entries=150, sequenceid=475, filesize=30.5 K 2024-11-11T23:56:10,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/acb11c68d19444bcac569523211b7ec1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/acb11c68d19444bcac569523211b7ec1 2024-11-11T23:56:10,973 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/acb11c68d19444bcac569523211b7ec1, entries=150, sequenceid=475, filesize=12.0 K 2024-11-11T23:56:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/47f6ba4a0e9144f3890b186b6380d4aa as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/47f6ba4a0e9144f3890b186b6380d4aa 2024-11-11T23:56:10,976 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/47f6ba4a0e9144f3890b186b6380d4aa, entries=150, sequenceid=475, filesize=12.0 K 2024-11-11T23:56:10,977 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 25afeec37b6a72df3a36de94b90eba91 in 1668ms, sequenceid=475, compaction requested=false 2024-11-11T23:56:10,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:10,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:10,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-11T23:56:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-11T23:56:10,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T23:56:10,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4300 sec 2024-11-11T23:56:10,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.4340 sec 2024-11-11T23:56:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:11,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T23:56:11,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:11,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:11,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:11,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:11,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:11,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:11,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ee0d61906c1f4ce383e31cdfb593266f_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369371405/Put/seqid=0 2024-11-11T23:56:11,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742398_1574 (size=12454) 2024-11-11T23:56:11,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369431440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:11,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369431555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:11,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369431764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:11,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:11,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ee0d61906c1f4ce383e31cdfb593266f_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ee0d61906c1f4ce383e31cdfb593266f_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:11,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/0a635934433e4ba6bf4c9bb74ba0b617, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:11,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/0a635934433e4ba6bf4c9bb74ba0b617 is 175, key is test_row_0/A:col10/1731369371405/Put/seqid=0 2024-11-11T23:56:11,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742399_1575 (size=31255) 2024-11-11T23:56:12,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369432075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:12,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=498, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/0a635934433e4ba6bf4c9bb74ba0b617 2024-11-11T23:56:12,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/a6c2c63cf42f41928322eeea3350958c is 50, key is test_row_0/B:col10/1731369371405/Put/seqid=0 2024-11-11T23:56:12,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742400_1576 (size=12301) 2024-11-11T23:56:12,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369432584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:12,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/a6c2c63cf42f41928322eeea3350958c 2024-11-11T23:56:12,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/20b3f5b6e91f4c89b60e7422841604b9 is 50, key is test_row_0/C:col10/1731369371405/Put/seqid=0 2024-11-11T23:56:12,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742401_1577 (size=12301) 2024-11-11T23:56:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54474 deadline: 1731369432714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:12,717 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_1' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T23:56:12,725 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:56:12,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T23:56:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T23:56:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T23:56:12,727 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T23:56:12,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731369432724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:12,727 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T23:56:12,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T23:56:12,727 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_2' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54476 deadline: 1731369432754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:12,757 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:12,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731369432764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:12,767 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b70d29849faf,42393,1731369201430 for 'test_row_0' in 25afeec37b6a72df3a36de94b90eba91 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T23:56:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T23:56:12,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T23:56:12,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:12,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:12,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:12,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:12,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:13,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T23:56:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:13,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:13,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/20b3f5b6e91f4c89b60e7422841604b9 2024-11-11T23:56:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T23:56:13,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/0a635934433e4ba6bf4c9bb74ba0b617 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/0a635934433e4ba6bf4c9bb74ba0b617 2024-11-11T23:56:13,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/0a635934433e4ba6bf4c9bb74ba0b617, entries=150, sequenceid=498, filesize=30.5 K 2024-11-11T23:56:13,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/a6c2c63cf42f41928322eeea3350958c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/a6c2c63cf42f41928322eeea3350958c 2024-11-11T23:56:13,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/a6c2c63cf42f41928322eeea3350958c, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T23:56:13,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/20b3f5b6e91f4c89b60e7422841604b9 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/20b3f5b6e91f4c89b60e7422841604b9 2024-11-11T23:56:13,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/20b3f5b6e91f4c89b60e7422841604b9, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T23:56:13,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 25afeec37b6a72df3a36de94b90eba91 in 1650ms, sequenceid=498, compaction requested=true 2024-11-11T23:56:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:56:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:13,056 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:56:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:13,056 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:56:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:13,057 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:13,057 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:13,057 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/B is initiating minor compaction (all files) 2024-11-11T23:56:13,057 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/A is initiating minor compaction (all files) 2024-11-11T23:56:13,057 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/B in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:13,057 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/A in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:13,057 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8b2dd23549c24afca819bd6cfb293075, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/0a635934433e4ba6bf4c9bb74ba0b617] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=92.1 K 2024-11-11T23:56:13,057 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/02026b67d8764964a5362971b25479a7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/acb11c68d19444bcac569523211b7ec1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/a6c2c63cf42f41928322eeea3350958c] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=36.5 K 2024-11-11T23:56:13,057 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:13,057 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8b2dd23549c24afca819bd6cfb293075, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/0a635934433e4ba6bf4c9bb74ba0b617] 2024-11-11T23:56:13,058 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 02026b67d8764964a5362971b25479a7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731369367636 2024-11-11T23:56:13,058 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b2dd23549c24afca819bd6cfb293075, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731369367636 2024-11-11T23:56:13,058 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting acb11c68d19444bcac569523211b7ec1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731369368030 2024-11-11T23:56:13,058 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bebf6d8f32c4ba9ab10c4ce4d9c3eee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731369368030 2024-11-11T23:56:13,058 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a635934433e4ba6bf4c9bb74ba0b617, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369370254 2024-11-11T23:56:13,058 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c2c63cf42f41928322eeea3350958c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369370254 2024-11-11T23:56:13,062 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:13,064 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#B#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:13,064 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/99dc3ee2dd99440b81462908b60eeb22 is 50, key is test_row_0/B:col10/1731369371405/Put/seqid=0 2024-11-11T23:56:13,068 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e51e54410f7743129d8ff1285cd11f3b_25afeec37b6a72df3a36de94b90eba91 store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742402_1578 (size=12915) 2024-11-11T23:56:13,069 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e51e54410f7743129d8ff1285cd11f3b_25afeec37b6a72df3a36de94b90eba91, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:13,069 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e51e54410f7743129d8ff1285cd11f3b_25afeec37b6a72df3a36de94b90eba91 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:13,075 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/99dc3ee2dd99440b81462908b60eeb22 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/99dc3ee2dd99440b81462908b60eeb22 2024-11-11T23:56:13,079 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/B of 25afeec37b6a72df3a36de94b90eba91 into 99dc3ee2dd99440b81462908b60eeb22(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:13,079 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:13,079 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/B, priority=13, startTime=1731369373056; duration=0sec 2024-11-11T23:56:13,079 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:13,079 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:B 2024-11-11T23:56:13,079 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:13,080 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:13,080 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/C is initiating minor compaction (all files) 2024-11-11T23:56:13,080 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/C in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:13,080 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8613b207880540c7a81eaefd4ba8cf25, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/47f6ba4a0e9144f3890b186b6380d4aa, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/20b3f5b6e91f4c89b60e7422841604b9] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=36.5 K 2024-11-11T23:56:13,081 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8613b207880540c7a81eaefd4ba8cf25, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731369367636 2024-11-11T23:56:13,081 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f6ba4a0e9144f3890b186b6380d4aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731369368030 2024-11-11T23:56:13,081 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b3f5b6e91f4c89b60e7422841604b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369370254 2024-11-11T23:56:13,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742403_1579 (size=4469) 2024-11-11T23:56:13,089 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#C#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:13,089 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/a6c5a190918b43f5b7e46c8e2f54a858 is 50, key is test_row_0/C:col10/1731369371405/Put/seqid=0 2024-11-11T23:56:13,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742404_1580 (size=12915) 2024-11-11T23:56:13,105 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/a6c5a190918b43f5b7e46c8e2f54a858 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/a6c5a190918b43f5b7e46c8e2f54a858 2024-11-11T23:56:13,108 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/C of 25afeec37b6a72df3a36de94b90eba91 into a6c5a190918b43f5b7e46c8e2f54a858(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:13,108 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:13,108 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/C, priority=13, startTime=1731369373056; duration=0sec 2024-11-11T23:56:13,108 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:13,108 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:C 2024-11-11T23:56:13,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T23:56:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:13,183 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T23:56:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:13,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111160d752bc699941d1ba15c0ba9aa4ef26_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369371436/Put/seqid=0 2024-11-11T23:56:13,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742405_1581 (size=12454) 2024-11-11T23:56:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T23:56:13,484 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:13,484 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/1a42e0c105874123bb829959d64acdf1 is 175, key is test_row_0/A:col10/1731369371405/Put/seqid=0 2024-11-11T23:56:13,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742406_1582 (size=31869) 2024-11-11T23:56:13,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:13,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. as already flushing 2024-11-11T23:56:13,599 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111160d752bc699941d1ba15c0ba9aa4ef26_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111160d752bc699941d1ba15c0ba9aa4ef26_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/12f51249321d495ab73205d8f66c5e18, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/12f51249321d495ab73205d8f66c5e18 is 175, key is test_row_0/A:col10/1731369371436/Put/seqid=0 2024-11-11T23:56:13,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742407_1583 (size=31255) 2024-11-11T23:56:13,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369433669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:13,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369433785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T23:56:13,890 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/1a42e0c105874123bb829959d64acdf1 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/1a42e0c105874123bb829959d64acdf1 2024-11-11T23:56:13,894 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/A of 25afeec37b6a72df3a36de94b90eba91 into 1a42e0c105874123bb829959d64acdf1(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:13,894 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:13,894 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/A, priority=13, startTime=1731369373056; duration=0sec 2024-11-11T23:56:13,894 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:13,894 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:A 2024-11-11T23:56:13,918 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-11-11T23:56:13,918 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:13,918 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,918 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:13,919 INFO [Thread-2276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2276. 2024-11-11T23:56:13,919 DEBUG [Thread-2276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:13,919 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,919 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:13,921 INFO [Thread-2282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2282. 2024-11-11T23:56:13,921 DEBUG [Thread-2282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:13,921 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,921 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,921 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-11-11T23:56:13,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:13,921 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:13,921 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,921 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:13,923 INFO [Thread-2284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2284. 2024-11-11T23:56:13,923 DEBUG [Thread-2284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:13,923 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,923 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:13,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:13,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369433995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:13,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:13,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:14,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=512, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/12f51249321d495ab73205d8f66c5e18 2024-11-11T23:56:14,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/8c2bd24eacab4a6182f6f7e14c1a56ee is 50, key is test_row_0/B:col10/1731369371436/Put/seqid=0 2024-11-11T23:56:14,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742408_1584 (size=12301) 2024-11-11T23:56:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369434305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:14,423 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/8c2bd24eacab4a6182f6f7e14c1a56ee 2024-11-11T23:56:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/dac0fa4d1f29491f91296eaa74b71be5 is 50, key is test_row_0/C:col10/1731369371436/Put/seqid=0 2024-11-11T23:56:14,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742409_1585 (size=12301) 2024-11-11T23:56:14,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T23:56:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731369434816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 2024-11-11T23:56:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 , the old value is region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T23:56:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., hostname=b70d29849faf,42393,1731369201430, seqNum=5 because the exception is null or not the one we care about 2024-11-11T23:56:14,840 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/dac0fa4d1f29491f91296eaa74b71be5 2024-11-11T23:56:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/12f51249321d495ab73205d8f66c5e18 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/12f51249321d495ab73205d8f66c5e18 2024-11-11T23:56:14,856 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/12f51249321d495ab73205d8f66c5e18, entries=150, sequenceid=512, filesize=30.5 K 2024-11-11T23:56:14,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/8c2bd24eacab4a6182f6f7e14c1a56ee as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8c2bd24eacab4a6182f6f7e14c1a56ee 2024-11-11T23:56:14,861 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8c2bd24eacab4a6182f6f7e14c1a56ee, entries=150, sequenceid=512, filesize=12.0 K 2024-11-11T23:56:14,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/dac0fa4d1f29491f91296eaa74b71be5 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/dac0fa4d1f29491f91296eaa74b71be5 2024-11-11T23:56:14,866 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/dac0fa4d1f29491f91296eaa74b71be5, entries=150, sequenceid=512, filesize=12.0 K 2024-11-11T23:56:14,867 INFO [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 25afeec37b6a72df3a36de94b90eba91 in 1683ms, sequenceid=512, compaction requested=false 2024-11-11T23:56:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T23:56:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T23:56:14,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T23:56:14,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1410 sec 2024-11-11T23:56:14,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.1460 sec 2024-11-11T23:56:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T23:56:14,874 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:56:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42393 {}] regionserver.HRegion(8855): Flush requested on 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:15,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T23:56:15,830 INFO [Thread-2271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2271. 2024-11-11T23:56:15,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:15,830 DEBUG [Thread-2271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:15,831 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:15,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:56:15,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:15,831 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:15,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:56:15,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:15,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:15,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T23:56:15,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b94fcb7cb26e41b5a8fda2cdb284659c_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369373634/Put/seqid=0 2024-11-11T23:56:15,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742410_1586 (size=12454) 2024-11-11T23:56:16,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:16,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b94fcb7cb26e41b5a8fda2cdb284659c_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b94fcb7cb26e41b5a8fda2cdb284659c_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:16,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/b86d5759de0b45558349704699190207, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:16,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/b86d5759de0b45558349704699190207 is 175, key is test_row_0/A:col10/1731369373634/Put/seqid=0 2024-11-11T23:56:16,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742411_1587 (size=31255) 2024-11-11T23:56:16,657 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=534, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/b86d5759de0b45558349704699190207 2024-11-11T23:56:16,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/2bbc5d48079c4342b6e36e99637729fb is 50, key is test_row_0/B:col10/1731369373634/Put/seqid=0 2024-11-11T23:56:16,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742412_1588 (size=12301) 2024-11-11T23:56:17,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/2bbc5d48079c4342b6e36e99637729fb 2024-11-11T23:56:17,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/645cd55bd1894d999a5f308d7be2c4aa is 50, key is test_row_0/C:col10/1731369373634/Put/seqid=0 2024-11-11T23:56:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742413_1589 (size=12301) 2024-11-11T23:56:17,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/645cd55bd1894d999a5f308d7be2c4aa 2024-11-11T23:56:17,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/b86d5759de0b45558349704699190207 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/b86d5759de0b45558349704699190207 2024-11-11T23:56:17,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/b86d5759de0b45558349704699190207, entries=150, sequenceid=534, filesize=30.5 K 2024-11-11T23:56:17,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/2bbc5d48079c4342b6e36e99637729fb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/2bbc5d48079c4342b6e36e99637729fb 2024-11-11T23:56:17,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/2bbc5d48079c4342b6e36e99637729fb, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T23:56:17,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/645cd55bd1894d999a5f308d7be2c4aa as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/645cd55bd1894d999a5f308d7be2c4aa 2024-11-11T23:56:17,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/645cd55bd1894d999a5f308d7be2c4aa, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T23:56:17,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for 25afeec37b6a72df3a36de94b90eba91 in 1680ms, sequenceid=534, compaction requested=true 2024-11-11T23:56:17,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:17,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T23:56:17,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:17,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T23:56:17,509 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:17,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:17,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25afeec37b6a72df3a36de94b90eba91:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T23:56:17,509 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:17,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:17,510 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:17,510 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:17,510 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/B is initiating minor compaction (all files) 2024-11-11T23:56:17,510 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/A is initiating minor compaction (all files) 2024-11-11T23:56:17,510 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/B in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:17,510 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/A in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:17,511 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/99dc3ee2dd99440b81462908b60eeb22, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8c2bd24eacab4a6182f6f7e14c1a56ee, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/2bbc5d48079c4342b6e36e99637729fb] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=36.6 K 2024-11-11T23:56:17,511 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/1a42e0c105874123bb829959d64acdf1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/12f51249321d495ab73205d8f66c5e18, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/b86d5759de0b45558349704699190207] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=92.2 K 2024-11-11T23:56:17,511 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:17,511 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. files: [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/1a42e0c105874123bb829959d64acdf1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/12f51249321d495ab73205d8f66c5e18, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/b86d5759de0b45558349704699190207] 2024-11-11T23:56:17,511 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 99dc3ee2dd99440b81462908b60eeb22, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369370254 2024-11-11T23:56:17,511 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a42e0c105874123bb829959d64acdf1, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369370254 2024-11-11T23:56:17,511 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c2bd24eacab4a6182f6f7e14c1a56ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731369371432 2024-11-11T23:56:17,511 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f51249321d495ab73205d8f66c5e18, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731369371432 2024-11-11T23:56:17,512 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbc5d48079c4342b6e36e99637729fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731369373634 2024-11-11T23:56:17,512 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b86d5759de0b45558349704699190207, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731369373634 2024-11-11T23:56:17,520 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:17,520 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/648694f4ad2145659d5dba2840b1a95a is 50, key is test_row_0/B:col10/1731369373634/Put/seqid=0 2024-11-11T23:56:17,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742414_1590 (size=13017) 2024-11-11T23:56:17,524 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:17,526 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118f6b75417d0f4ccd814d7f8f334bdee3_25afeec37b6a72df3a36de94b90eba91 store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:17,528 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/648694f4ad2145659d5dba2840b1a95a as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/648694f4ad2145659d5dba2840b1a95a 2024-11-11T23:56:17,533 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/B of 25afeec37b6a72df3a36de94b90eba91 into 648694f4ad2145659d5dba2840b1a95a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:17,533 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:17,533 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/B, priority=13, startTime=1731369377509; duration=0sec 2024-11-11T23:56:17,533 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T23:56:17,533 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:B 2024-11-11T23:56:17,533 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T23:56:17,537 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T23:56:17,537 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1541): 25afeec37b6a72df3a36de94b90eba91/C is initiating minor compaction (all files) 2024-11-11T23:56:17,537 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25afeec37b6a72df3a36de94b90eba91/C in TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:17,537 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/a6c5a190918b43f5b7e46c8e2f54a858, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/dac0fa4d1f29491f91296eaa74b71be5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/645cd55bd1894d999a5f308d7be2c4aa] into tmpdir=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp, totalSize=36.6 K 2024-11-11T23:56:17,538 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c5a190918b43f5b7e46c8e2f54a858, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731369370254 2024-11-11T23:56:17,538 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting dac0fa4d1f29491f91296eaa74b71be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731369371432 2024-11-11T23:56:17,538 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] compactions.Compactor(225): Compacting 645cd55bd1894d999a5f308d7be2c4aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731369373634 2024-11-11T23:56:17,540 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118f6b75417d0f4ccd814d7f8f334bdee3_25afeec37b6a72df3a36de94b90eba91, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:17,540 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118f6b75417d0f4ccd814d7f8f334bdee3_25afeec37b6a72df3a36de94b90eba91 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:17,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742415_1591 (size=4469) 2024-11-11T23:56:17,549 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#A#compaction#502 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:17,550 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/9da35381dbdc46ec801f5250f5d448f3 is 175, key is test_row_0/A:col10/1731369373634/Put/seqid=0 2024-11-11T23:56:17,553 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25afeec37b6a72df3a36de94b90eba91#C#compaction#503 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T23:56:17,553 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/186cbb24a58546318420d416e7e4a102 is 50, key is test_row_0/C:col10/1731369373634/Put/seqid=0 2024-11-11T23:56:17,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742416_1592 (size=31971) 2024-11-11T23:56:17,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742417_1593 (size=13017) 2024-11-11T23:56:17,564 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/9da35381dbdc46ec801f5250f5d448f3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9da35381dbdc46ec801f5250f5d448f3 2024-11-11T23:56:17,566 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/186cbb24a58546318420d416e7e4a102 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/186cbb24a58546318420d416e7e4a102 2024-11-11T23:56:17,568 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/A of 25afeec37b6a72df3a36de94b90eba91 into 9da35381dbdc46ec801f5250f5d448f3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:17,568 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:17,568 INFO [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/A, priority=13, startTime=1731369377509; duration=0sec 2024-11-11T23:56:17,568 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:17,568 DEBUG [RS:0;b70d29849faf:42393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:A 2024-11-11T23:56:17,570 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25afeec37b6a72df3a36de94b90eba91/C of 25afeec37b6a72df3a36de94b90eba91 into 186cbb24a58546318420d416e7e4a102(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T23:56:17,570 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25afeec37b6a72df3a36de94b90eba91: 2024-11-11T23:56:17,570 INFO [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., storeName=25afeec37b6a72df3a36de94b90eba91/C, priority=13, startTime=1731369377509; duration=0sec 2024-11-11T23:56:17,570 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T23:56:17,570 DEBUG [RS:0;b70d29849faf:42393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25afeec37b6a72df3a36de94b90eba91:C 2024-11-11T23:56:19,732 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:56:22,759 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-11-11T23:56:22,759 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:22,759 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,759 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:22,807 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-11-11T23:56:22,808 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:22,808 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,808 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:22,826 INFO [Thread-2265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2265. 2024-11-11T23:56:22,826 DEBUG [Thread-2265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:22,826 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,826 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:22,846 INFO [Thread-2269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2269. 2024-11-11T23:56:22,846 DEBUG [Thread-2269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T23:56:22,846 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,846 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 167 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3383 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3247 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3274 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3374 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3247 2024-11-11T23:56:22,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T23:56:22,846 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:56:22,846 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:56:22,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:22,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:22,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T23:56:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T23:56:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T23:56:22,852 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369382852"}]},"ts":"1731369382852"} 2024-11-11T23:56:22,854 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T23:56:22,854 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T23:56:22,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T23:56:22,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, UNASSIGN}] 2024-11-11T23:56:22,858 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, UNASSIGN 2024-11-11T23:56:22,859 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=25afeec37b6a72df3a36de94b90eba91, regionState=CLOSING, regionLocation=b70d29849faf,42393,1731369201430 2024-11-11T23:56:22,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, UNASSIGN because future has completed 2024-11-11T23:56:22,861 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T23:56:22,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430}] 2024-11-11T23:56:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T23:56:23,016 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(122): Close 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:23,016 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T23:56:23,016 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1722): Closing 25afeec37b6a72df3a36de94b90eba91, disabling compactions & flushes 2024-11-11T23:56:23,016 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:23,016 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:23,017 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. after waiting 0 ms 2024-11-11T23:56:23,017 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:23,017 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(2902): Flushing 25afeec37b6a72df3a36de94b90eba91 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T23:56:23,018 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=A 2024-11-11T23:56:23,018 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:23,018 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=B 2024-11-11T23:56:23,018 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:23,018 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 25afeec37b6a72df3a36de94b90eba91, store=C 2024-11-11T23:56:23,019 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T23:56:23,028 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111119c19586ebb64e31ab4056708d284c3e_25afeec37b6a72df3a36de94b90eba91 is 50, key is test_row_0/A:col10/1731369382844/Put/seqid=0 2024-11-11T23:56:23,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742418_1594 (size=12454) 2024-11-11T23:56:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T23:56:23,432 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:23,437 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111119c19586ebb64e31ab4056708d284c3e_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111119c19586ebb64e31ab4056708d284c3e_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:23,438 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/3ebfef3767414c1692b6127fd0a1a64b, store: [table=TestAcidGuarantees family=A region=25afeec37b6a72df3a36de94b90eba91] 2024-11-11T23:56:23,438 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/3ebfef3767414c1692b6127fd0a1a64b is 175, key is test_row_0/A:col10/1731369382844/Put/seqid=0 2024-11-11T23:56:23,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742419_1595 (size=31255) 2024-11-11T23:56:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T23:56:23,843 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=544, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/3ebfef3767414c1692b6127fd0a1a64b 2024-11-11T23:56:23,857 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/7acfc0c3a5694b4da29de082b2a8887c is 50, key is test_row_0/B:col10/1731369382844/Put/seqid=0 2024-11-11T23:56:23,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742420_1596 (size=12301) 2024-11-11T23:56:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T23:56:24,261 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/7acfc0c3a5694b4da29de082b2a8887c 2024-11-11T23:56:24,269 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/9e6196619cc34decba49ffc2327aa381 is 50, key is test_row_0/C:col10/1731369382844/Put/seqid=0 2024-11-11T23:56:24,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742421_1597 (size=12301) 2024-11-11T23:56:24,673 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/9e6196619cc34decba49ffc2327aa381 2024-11-11T23:56:24,676 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/A/3ebfef3767414c1692b6127fd0a1a64b as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3ebfef3767414c1692b6127fd0a1a64b 2024-11-11T23:56:24,679 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3ebfef3767414c1692b6127fd0a1a64b, entries=150, sequenceid=544, filesize=30.5 K 2024-11-11T23:56:24,680 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/B/7acfc0c3a5694b4da29de082b2a8887c as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/7acfc0c3a5694b4da29de082b2a8887c 2024-11-11T23:56:24,683 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/7acfc0c3a5694b4da29de082b2a8887c, entries=150, sequenceid=544, filesize=12.0 K 2024-11-11T23:56:24,683 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/.tmp/C/9e6196619cc34decba49ffc2327aa381 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/9e6196619cc34decba49ffc2327aa381 2024-11-11T23:56:24,685 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/9e6196619cc34decba49ffc2327aa381, entries=150, sequenceid=544, filesize=12.0 K 2024-11-11T23:56:24,686 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 25afeec37b6a72df3a36de94b90eba91 in 1669ms, sequenceid=544, compaction requested=false 2024-11-11T23:56:24,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/2beaf225d8244372a53caf2e7b45163b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d0a971a32aec4a7ebeae22050ad0199c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/54fef06bb67545319f0c3485bec616e6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d4a8bb1c50e44329b67c49d1bcb7394a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/e7e6bf8957704ef89a27eb9df0964f30, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/965c85a7d441423a8eba8979c77d7fc2, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/ff91f1b7f4ff4d3181b5febcb221edc6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/050b36beaefc42e38b5fc0c24c4c4a69, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/28c84d802d0e475391d516a872f8f3f9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/bb0fbe935a224913a48ae895ca38c0cd, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3e0c8e7241094c0b9f2b7f46401b1697, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/7b5fc4945fc44782abbec7d095653d51, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/453eef3f409a4bb7bbc1608943646d56, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8e1eb7a8ac6f4dcb8b49905b03615c63, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/50f3aa8294e84e29b3867900aa624065, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8b2dd23549c24afca819bd6cfb293075, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/1a42e0c105874123bb829959d64acdf1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/0a635934433e4ba6bf4c9bb74ba0b617, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/12f51249321d495ab73205d8f66c5e18, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/b86d5759de0b45558349704699190207] to archive 2024-11-11T23:56:24,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:56:24,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/2beaf225d8244372a53caf2e7b45163b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/2beaf225d8244372a53caf2e7b45163b 2024-11-11T23:56:24,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d0a971a32aec4a7ebeae22050ad0199c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d0a971a32aec4a7ebeae22050ad0199c 2024-11-11T23:56:24,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/54fef06bb67545319f0c3485bec616e6 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/54fef06bb67545319f0c3485bec616e6 2024-11-11T23:56:24,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d4a8bb1c50e44329b67c49d1bcb7394a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/d4a8bb1c50e44329b67c49d1bcb7394a 2024-11-11T23:56:24,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/e7e6bf8957704ef89a27eb9df0964f30 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/e7e6bf8957704ef89a27eb9df0964f30 2024-11-11T23:56:24,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/965c85a7d441423a8eba8979c77d7fc2 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/965c85a7d441423a8eba8979c77d7fc2 2024-11-11T23:56:24,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/ff91f1b7f4ff4d3181b5febcb221edc6 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/ff91f1b7f4ff4d3181b5febcb221edc6 2024-11-11T23:56:24,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/050b36beaefc42e38b5fc0c24c4c4a69 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/050b36beaefc42e38b5fc0c24c4c4a69 2024-11-11T23:56:24,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/28c84d802d0e475391d516a872f8f3f9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/28c84d802d0e475391d516a872f8f3f9 2024-11-11T23:56:24,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/bb0fbe935a224913a48ae895ca38c0cd to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/bb0fbe935a224913a48ae895ca38c0cd 2024-11-11T23:56:24,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3e0c8e7241094c0b9f2b7f46401b1697 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3e0c8e7241094c0b9f2b7f46401b1697 2024-11-11T23:56:24,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/7b5fc4945fc44782abbec7d095653d51 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/7b5fc4945fc44782abbec7d095653d51 2024-11-11T23:56:24,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/453eef3f409a4bb7bbc1608943646d56 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/453eef3f409a4bb7bbc1608943646d56 2024-11-11T23:56:24,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8e1eb7a8ac6f4dcb8b49905b03615c63 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8e1eb7a8ac6f4dcb8b49905b03615c63 2024-11-11T23:56:24,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/50f3aa8294e84e29b3867900aa624065 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/50f3aa8294e84e29b3867900aa624065 2024-11-11T23:56:24,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8b2dd23549c24afca819bd6cfb293075 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/8b2dd23549c24afca819bd6cfb293075 2024-11-11T23:56:24,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9bebf6d8f32c4ba9ab10c4ce4d9c3eee 2024-11-11T23:56:24,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/1a42e0c105874123bb829959d64acdf1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/1a42e0c105874123bb829959d64acdf1 2024-11-11T23:56:24,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/0a635934433e4ba6bf4c9bb74ba0b617 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/0a635934433e4ba6bf4c9bb74ba0b617 2024-11-11T23:56:24,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/12f51249321d495ab73205d8f66c5e18 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/12f51249321d495ab73205d8f66c5e18 2024-11-11T23:56:24,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/b86d5759de0b45558349704699190207 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/b86d5759de0b45558349704699190207 2024-11-11T23:56:24,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f3220f7c58b243efa22ec2d82e3b4c25, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/d87bc71472e241159709a572e49d339a, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/eee6912e7c244fe9af80e325e9881bd8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/1b53009cb22f4f38af41034edc336407, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/fd0e66b760df4420ac8a35a2820347ea, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/733aa45390a748a6a58ac7fc0ac841e9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f89f58a3f58846878a6da646eae8a795, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e41a2678b3714a3b976df0a80d9ddc6b, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/bd39fa31d7494fb8819c7d576c2107af, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/857f886d1f3e4bc4a4a7a7e6437369f6, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e39b0276ab2a49d5ac43b66b8d184ea0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8f1dca878cbd42d0a0100e4ae67170c8, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/cbb28a374a314ad38511bcf1b599f2c3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/febf8b984aa8442da2199bcacaa7e270, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/02026b67d8764964a5362971b25479a7, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/ae327435777d4cc887303d5d8793bf8c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/acb11c68d19444bcac569523211b7ec1, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/99dc3ee2dd99440b81462908b60eeb22, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/a6c2c63cf42f41928322eeea3350958c, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8c2bd24eacab4a6182f6f7e14c1a56ee, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/2bbc5d48079c4342b6e36e99637729fb] to archive 2024-11-11T23:56:24,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:56:24,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f3220f7c58b243efa22ec2d82e3b4c25 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f3220f7c58b243efa22ec2d82e3b4c25 2024-11-11T23:56:24,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/d87bc71472e241159709a572e49d339a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/d87bc71472e241159709a572e49d339a 2024-11-11T23:56:24,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/eee6912e7c244fe9af80e325e9881bd8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/eee6912e7c244fe9af80e325e9881bd8 2024-11-11T23:56:24,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/1b53009cb22f4f38af41034edc336407 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/1b53009cb22f4f38af41034edc336407 2024-11-11T23:56:24,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/fd0e66b760df4420ac8a35a2820347ea to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/fd0e66b760df4420ac8a35a2820347ea 2024-11-11T23:56:24,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/733aa45390a748a6a58ac7fc0ac841e9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/733aa45390a748a6a58ac7fc0ac841e9 2024-11-11T23:56:24,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f89f58a3f58846878a6da646eae8a795 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/f89f58a3f58846878a6da646eae8a795 2024-11-11T23:56:24,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e41a2678b3714a3b976df0a80d9ddc6b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e41a2678b3714a3b976df0a80d9ddc6b 2024-11-11T23:56:24,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/bd39fa31d7494fb8819c7d576c2107af to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/bd39fa31d7494fb8819c7d576c2107af 2024-11-11T23:56:24,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/857f886d1f3e4bc4a4a7a7e6437369f6 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/857f886d1f3e4bc4a4a7a7e6437369f6 2024-11-11T23:56:24,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e39b0276ab2a49d5ac43b66b8d184ea0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/e39b0276ab2a49d5ac43b66b8d184ea0 2024-11-11T23:56:24,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8f1dca878cbd42d0a0100e4ae67170c8 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8f1dca878cbd42d0a0100e4ae67170c8 2024-11-11T23:56:24,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/cbb28a374a314ad38511bcf1b599f2c3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/cbb28a374a314ad38511bcf1b599f2c3 2024-11-11T23:56:24,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/febf8b984aa8442da2199bcacaa7e270 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/febf8b984aa8442da2199bcacaa7e270 2024-11-11T23:56:24,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/02026b67d8764964a5362971b25479a7 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/02026b67d8764964a5362971b25479a7 2024-11-11T23:56:24,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/ae327435777d4cc887303d5d8793bf8c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/ae327435777d4cc887303d5d8793bf8c 2024-11-11T23:56:24,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/acb11c68d19444bcac569523211b7ec1 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/acb11c68d19444bcac569523211b7ec1 2024-11-11T23:56:24,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/99dc3ee2dd99440b81462908b60eeb22 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/99dc3ee2dd99440b81462908b60eeb22 2024-11-11T23:56:24,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/a6c2c63cf42f41928322eeea3350958c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/a6c2c63cf42f41928322eeea3350958c 2024-11-11T23:56:24,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8c2bd24eacab4a6182f6f7e14c1a56ee to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/8c2bd24eacab4a6182f6f7e14c1a56ee 2024-11-11T23:56:24,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/2bbc5d48079c4342b6e36e99637729fb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/2bbc5d48079c4342b6e36e99637729fb 2024-11-11T23:56:24,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2b87b943658446e2974dddf3a1d536ef, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/da3bb0a943804afbaf534cfc3f123099, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/81f6c9400d39421f84ac1d15df118b87, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/faa4b1b2299741e78810c8c05df43425, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/46e6d9089a084c558917c93903150835, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/d83235d71be64dcea1f9ead422dab848, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/c8f9bedaeca945b29e1dbee05abe00fc, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/abb75a9f0d374da6b7789cea26d8b4f5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4581d08da4084754a1c0075435499e79, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8daafab018ba44eaa4d4ba8723e70854, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2fef02c4a7dd4a7ab1ef9600ae2fea06, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8707b8c5fbc54fd8b6605e83534e8bcb, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/f2e43be3ff834a369e8f5636792b0fc3, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/88d19d523601491ea4c44082bddf0e57, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8613b207880540c7a81eaefd4ba8cf25, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4b395b7c1f384b0e989be9e66b5e67e0, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/47f6ba4a0e9144f3890b186b6380d4aa, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/a6c5a190918b43f5b7e46c8e2f54a858, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/20b3f5b6e91f4c89b60e7422841604b9, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/dac0fa4d1f29491f91296eaa74b71be5, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/645cd55bd1894d999a5f308d7be2c4aa] to archive 2024-11-11T23:56:24,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T23:56:24,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2b87b943658446e2974dddf3a1d536ef to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2b87b943658446e2974dddf3a1d536ef 2024-11-11T23:56:24,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/da3bb0a943804afbaf534cfc3f123099 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/da3bb0a943804afbaf534cfc3f123099 2024-11-11T23:56:24,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/81f6c9400d39421f84ac1d15df118b87 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/81f6c9400d39421f84ac1d15df118b87 2024-11-11T23:56:24,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/faa4b1b2299741e78810c8c05df43425 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/faa4b1b2299741e78810c8c05df43425 2024-11-11T23:56:24,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/46e6d9089a084c558917c93903150835 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/46e6d9089a084c558917c93903150835 2024-11-11T23:56:24,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/d83235d71be64dcea1f9ead422dab848 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/d83235d71be64dcea1f9ead422dab848 2024-11-11T23:56:24,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/c8f9bedaeca945b29e1dbee05abe00fc to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/c8f9bedaeca945b29e1dbee05abe00fc 2024-11-11T23:56:24,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/abb75a9f0d374da6b7789cea26d8b4f5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/abb75a9f0d374da6b7789cea26d8b4f5 2024-11-11T23:56:24,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4581d08da4084754a1c0075435499e79 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4581d08da4084754a1c0075435499e79 2024-11-11T23:56:24,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8daafab018ba44eaa4d4ba8723e70854 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8daafab018ba44eaa4d4ba8723e70854 2024-11-11T23:56:24,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2fef02c4a7dd4a7ab1ef9600ae2fea06 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/2fef02c4a7dd4a7ab1ef9600ae2fea06 2024-11-11T23:56:24,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8707b8c5fbc54fd8b6605e83534e8bcb to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8707b8c5fbc54fd8b6605e83534e8bcb 2024-11-11T23:56:24,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/f2e43be3ff834a369e8f5636792b0fc3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/f2e43be3ff834a369e8f5636792b0fc3 2024-11-11T23:56:24,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/88d19d523601491ea4c44082bddf0e57 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/88d19d523601491ea4c44082bddf0e57 2024-11-11T23:56:24,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8613b207880540c7a81eaefd4ba8cf25 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/8613b207880540c7a81eaefd4ba8cf25 2024-11-11T23:56:24,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4b395b7c1f384b0e989be9e66b5e67e0 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/4b395b7c1f384b0e989be9e66b5e67e0 2024-11-11T23:56:24,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/47f6ba4a0e9144f3890b186b6380d4aa to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/47f6ba4a0e9144f3890b186b6380d4aa 2024-11-11T23:56:24,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/a6c5a190918b43f5b7e46c8e2f54a858 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/a6c5a190918b43f5b7e46c8e2f54a858 2024-11-11T23:56:24,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/20b3f5b6e91f4c89b60e7422841604b9 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/20b3f5b6e91f4c89b60e7422841604b9 2024-11-11T23:56:24,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/dac0fa4d1f29491f91296eaa74b71be5 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/dac0fa4d1f29491f91296eaa74b71be5 2024-11-11T23:56:24,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/645cd55bd1894d999a5f308d7be2c4aa to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/645cd55bd1894d999a5f308d7be2c4aa 2024-11-11T23:56:24,746 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=4 2024-11-11T23:56:24,746 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. 2024-11-11T23:56:24,746 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1676): Region close journal for 25afeec37b6a72df3a36de94b90eba91: Waiting for close lock at 1731369383016Running coprocessor pre-close hooks at 1731369383016Disabling compacts and flushes for region at 1731369383016Disabling writes for close at 1731369383017 (+1 ms)Obtaining lock to block concurrent updates at 1731369383017Preparing flush snapshotting stores in 25afeec37b6a72df3a36de94b90eba91 at 1731369383017Finished memstore snapshotting TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731369383019 (+2 ms)Flushing stores of TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91. at 1731369383020 (+1 ms)Flushing 25afeec37b6a72df3a36de94b90eba91/A: creating writer at 1731369383020Flushing mob file 25afeec37b6a72df3a36de94b90eba91/A: appending metadata at 1731369383028 (+8 ms)Flushing mob file 25afeec37b6a72df3a36de94b90eba91/A: closing flushed file at 1731369383028Flushing 25afeec37b6a72df3a36de94b90eba91/A: appending metadata at 1731369383438 (+410 ms)Flushing 25afeec37b6a72df3a36de94b90eba91/A: closing flushed file at 1731369383438Flushing 25afeec37b6a72df3a36de94b90eba91/B: creating writer at 1731369383854 (+416 ms)Flushing 25afeec37b6a72df3a36de94b90eba91/B: appending metadata at 1731369383856 (+2 ms)Flushing 25afeec37b6a72df3a36de94b90eba91/B: closing flushed file at 1731369383856Flushing 25afeec37b6a72df3a36de94b90eba91/C: creating writer at 1731369384265 (+409 ms)Flushing 25afeec37b6a72df3a36de94b90eba91/C: appending metadata at 1731369384268 (+3 ms)Flushing 25afeec37b6a72df3a36de94b90eba91/C: closing flushed file at 1731369384268Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dfa0a8a: reopening flushed file at 1731369384676 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@134c5098: reopening flushed file at 1731369384679 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@682ffa8f: reopening flushed file at 1731369384683 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 25afeec37b6a72df3a36de94b90eba91 in 1669ms, sequenceid=544, compaction requested=false at 1731369384686 (+3 ms)Writing region close event to WAL at 1731369384743 (+57 ms)Running coprocessor post-close hooks at 1731369384746 (+3 ms)Closed at 1731369384746 2024-11-11T23:56:24,748 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(157): Closed 25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:24,748 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=25afeec37b6a72df3a36de94b90eba91, regionState=CLOSED 2024-11-11T23:56:24,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 because future has completed 2024-11-11T23:56:24,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T23:56:24,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseRegionProcedure 25afeec37b6a72df3a36de94b90eba91, server=b70d29849faf,42393,1731369201430 in 1.8890 sec 2024-11-11T23:56:24,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-11T23:56:24,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=25afeec37b6a72df3a36de94b90eba91, UNASSIGN in 1.8940 sec 2024-11-11T23:56:24,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T23:56:24,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8980 sec 2024-11-11T23:56:24,755 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369384755"}]},"ts":"1731369384755"} 2024-11-11T23:56:24,756 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T23:56:24,756 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T23:56:24,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-11T23:56:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T23:56:24,995 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:56:24,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T23:56:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:56:24,998 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:56:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T23:56:24,998 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=158, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:56:25,000 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,002 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C, FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/recovered.edits] 2024-11-11T23:56:25,003 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3ebfef3767414c1692b6127fd0a1a64b to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/3ebfef3767414c1692b6127fd0a1a64b 2024-11-11T23:56:25,004 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9da35381dbdc46ec801f5250f5d448f3 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/A/9da35381dbdc46ec801f5250f5d448f3 2024-11-11T23:56:25,006 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/648694f4ad2145659d5dba2840b1a95a to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/648694f4ad2145659d5dba2840b1a95a 2024-11-11T23:56:25,006 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/7acfc0c3a5694b4da29de082b2a8887c to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/B/7acfc0c3a5694b4da29de082b2a8887c 2024-11-11T23:56:25,008 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/186cbb24a58546318420d416e7e4a102 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/186cbb24a58546318420d416e7e4a102 2024-11-11T23:56:25,009 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/9e6196619cc34decba49ffc2327aa381 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/C/9e6196619cc34decba49ffc2327aa381 2024-11-11T23:56:25,011 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/recovered.edits/547.seqid to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91/recovered.edits/547.seqid 2024-11-11T23:56:25,011 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/default/TestAcidGuarantees/25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,011 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T23:56:25,012 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T23:56:25,012 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T23:56:25,015 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111119c19586ebb64e31ab4056708d284c3e_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111119c19586ebb64e31ab4056708d284c3e_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,016 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113714a482f8d24620a4f897dac71475c1_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113714a482f8d24620a4f897dac71475c1_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,016 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111144b878e8358a422982c448d1b3364210_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111144b878e8358a422982c448d1b3364210_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,017 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154160292d4ab4ac3bb9d086e0b4f1c89_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154160292d4ab4ac3bb9d086e0b4f1c89_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,018 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111160d752bc699941d1ba15c0ba9aa4ef26_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111160d752bc699941d1ba15c0ba9aa4ef26_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,019 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116a0a38d2de7b44d0b4fa9523aa49daf3_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116a0a38d2de7b44d0b4fa9523aa49daf3_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,020 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117b666caf68b041fca0b29ace0a14a482_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117b666caf68b041fca0b29ace0a14a482_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,021 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118357a69431274319bb289d929de4564d_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118357a69431274319bb289d929de4564d_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,022 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118991b580b85f407398030ab6df467128_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118991b580b85f407398030ab6df467128_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,022 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b94fcb7cb26e41b5a8fda2cdb284659c_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b94fcb7cb26e41b5a8fda2cdb284659c_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,023 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bbe972d2aa96404c81a59e296068eca7_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bbe972d2aa96404c81a59e296068eca7_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,024 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf317e4cd3c44296b49d67617551e5f2_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf317e4cd3c44296b49d67617551e5f2_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,025 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d77da99b20c6448bb60bc98447227f1b_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d77da99b20c6448bb60bc98447227f1b_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,025 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0f88ba102414d869ace2a2e97691df0_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0f88ba102414d869ace2a2e97691df0_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,026 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e449d1cbf3354adfa65c176f6b0ddeed_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e449d1cbf3354adfa65c176f6b0ddeed_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,027 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ea2e1f3389024503b95ebe23c3a7e868_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ea2e1f3389024503b95ebe23c3a7e868_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,028 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ee0d61906c1f4ce383e31cdfb593266f_25afeec37b6a72df3a36de94b90eba91 to hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ee0d61906c1f4ce383e31cdfb593266f_25afeec37b6a72df3a36de94b90eba91 2024-11-11T23:56:25,028 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T23:56:25,031 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=158, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:56:25,033 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T23:56:25,035 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T23:56:25,036 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=158, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:56:25,036 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T23:56:25,036 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731369385036"}]},"ts":"9223372036854775807"} 2024-11-11T23:56:25,038 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T23:56:25,038 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 25afeec37b6a72df3a36de94b90eba91, NAME => 'TestAcidGuarantees,,1731369350760.25afeec37b6a72df3a36de94b90eba91.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T23:56:25,038 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T23:56:25,038 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731369385038"}]},"ts":"9223372036854775807"} 2024-11-11T23:56:25,041 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T23:56:25,041 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=158, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T23:56:25,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-11T23:56:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T23:56:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T23:56:25,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T23:56:25,113 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 252), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 364), ProcessCount=11 (was 11), AvailableMemoryMB=4392 (was 4447) 2024-11-11T23:56:25,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T23:56:25,113 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:56:25,114 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:56:25,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:25,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:25,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:25,114 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T23:56:25,114 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=387043539, stopped=false 2024-11-11T23:56:25,114 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b70d29849faf,37327,1731369200577 2024-11-11T23:56:25,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T23:56:25,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T23:56:25,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:56:25,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:56:25,116 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T23:56:25,116 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:56:25,116 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:56:25,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:25,117 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b70d29849faf,42393,1731369201430' ***** 2024-11-11T23:56:25,117 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T23:56:25,117 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T23:56:25,117 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T23:56:25,118 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T23:56:25,118 INFO [RS:0;b70d29849faf:42393 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T23:56:25,118 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T23:56:25,118 INFO [RS:0;b70d29849faf:42393 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T23:56:25,118 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(959): stopping server b70d29849faf,42393,1731369201430 2024-11-11T23:56:25,119 INFO [RS:0;b70d29849faf:42393 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T23:56:25,119 INFO [RS:0;b70d29849faf:42393 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b70d29849faf:42393. 2024-11-11T23:56:25,119 DEBUG [RS:0;b70d29849faf:42393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:56:25,119 DEBUG [RS:0;b70d29849faf:42393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:25,119 INFO [RS:0;b70d29849faf:42393 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T23:56:25,119 INFO [RS:0;b70d29849faf:42393 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T23:56:25,119 INFO [RS:0;b70d29849faf:42393 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T23:56:25,119 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T23:56:25,120 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T23:56:25,120 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T23:56:25,121 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:25,121 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T23:56:25,121 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T23:56:25,121 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T23:56:25,121 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T23:56:25,121 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T23:56:25,121 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T23:56:25,134 INFO [regionserver/b70d29849faf:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T23:56:25,142 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/info/a7acc62b289a4452be0ec16402699ddb is 95, key is TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd./info:/1731369233343/DeleteFamily/seqid=0 2024-11-11T23:56:25,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742422_1598 (size=6514) 2024-11-11T23:56:25,321 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:25,521 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:25,546 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/info/a7acc62b289a4452be0ec16402699ddb 2024-11-11T23:56:25,563 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/ns/0344b27a3a254df994fdc76c1641cd6e is 93, key is TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd./ns:/1731369233299/DeleteFamily/seqid=0 2024-11-11T23:56:25,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742423_1599 (size=5906) 2024-11-11T23:56:25,583 INFO [regionserver/b70d29849faf:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T23:56:25,583 INFO [regionserver/b70d29849faf:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T23:56:25,721 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:25,922 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:25,966 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/ns/0344b27a3a254df994fdc76c1641cd6e 2024-11-11T23:56:25,985 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/rep_barrier/cff4655427ed4a7f807640c554377518 is 102, key is TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd./rep_barrier:/1731369233299/DeleteFamily/seqid=0 2024-11-11T23:56:25,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742424_1600 (size=6025) 2024-11-11T23:56:26,122 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:26,322 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:26,390 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/rep_barrier/cff4655427ed4a7f807640c554377518 2024-11-11T23:56:26,418 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/table/e9721648df744a35bcb93dd3acfe9af3 is 96, key is TestAcidGuarantees,,1731369204731.6b2061b8554b1e7422f6c037ae02cccd./table:/1731369233299/DeleteFamily/seqid=0 2024-11-11T23:56:26,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742425_1601 (size=5936) 2024-11-11T23:56:26,523 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:26,723 DEBUG [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T23:56:26,823 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/table/e9721648df744a35bcb93dd3acfe9af3 2024-11-11T23:56:26,832 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/info/a7acc62b289a4452be0ec16402699ddb as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/info/a7acc62b289a4452be0ec16402699ddb 2024-11-11T23:56:26,837 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/info/a7acc62b289a4452be0ec16402699ddb, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T23:56:26,838 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/ns/0344b27a3a254df994fdc76c1641cd6e as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/ns/0344b27a3a254df994fdc76c1641cd6e 2024-11-11T23:56:26,842 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/ns/0344b27a3a254df994fdc76c1641cd6e, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T23:56:26,842 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/rep_barrier/cff4655427ed4a7f807640c554377518 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/rep_barrier/cff4655427ed4a7f807640c554377518 2024-11-11T23:56:26,845 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/rep_barrier/cff4655427ed4a7f807640c554377518, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T23:56:26,846 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/.tmp/table/e9721648df744a35bcb93dd3acfe9af3 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/table/e9721648df744a35bcb93dd3acfe9af3 2024-11-11T23:56:26,849 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/table/e9721648df744a35bcb93dd3acfe9af3, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T23:56:26,850 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false 2024-11-11T23:56:26,855 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T23:56:26,855 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T23:56:26,856 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T23:56:26,856 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731369385120Running coprocessor pre-close hooks at 1731369385121 (+1 ms)Disabling compacts and flushes for region at 1731369385121Disabling writes for close at 1731369385121Obtaining lock to block concurrent updates at 1731369385121Preparing flush snapshotting stores in 1588230740 at 1731369385121Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731369385122 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731369385122Flushing 1588230740/info: creating writer at 1731369385123 (+1 ms)Flushing 1588230740/info: appending metadata at 1731369385142 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731369385142Flushing 1588230740/ns: creating writer at 1731369385549 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731369385562 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731369385562Flushing 1588230740/rep_barrier: creating writer at 1731369385969 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731369385985 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731369385985Flushing 1588230740/table: creating writer at 1731369386398 (+413 ms)Flushing 1588230740/table: appending metadata at 1731369386418 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731369386418Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40707c08: reopening flushed file at 1731369386830 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63cab24c: reopening flushed file at 1731369386838 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78d8a8bc: reopening flushed file at 1731369386842 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2273a78c: reopening flushed file at 1731369386846 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false at 1731369386850 (+4 ms)Writing region close event to WAL at 1731369386852 (+2 ms)Running coprocessor post-close hooks at 1731369386855 (+3 ms)Closed at 1731369386855 2024-11-11T23:56:26,856 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T23:56:26,923 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(976): stopping server b70d29849faf,42393,1731369201430; all regions closed. 2024-11-11T23:56:26,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741834_1010 (size=25344) 2024-11-11T23:56:26,932 DEBUG [RS:0;b70d29849faf:42393 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/oldWALs 2024-11-11T23:56:26,932 INFO [RS:0;b70d29849faf:42393 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b70d29849faf%2C42393%2C1731369201430.meta:.meta(num 1731369204271) 2024-11-11T23:56:26,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741833_1009 (size=20761318) 2024-11-11T23:56:26,937 DEBUG [RS:0;b70d29849faf:42393 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/oldWALs 2024-11-11T23:56:26,937 INFO [RS:0;b70d29849faf:42393 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b70d29849faf%2C42393%2C1731369201430:(num 1731369203681) 2024-11-11T23:56:26,937 DEBUG [RS:0;b70d29849faf:42393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:26,937 INFO [RS:0;b70d29849faf:42393 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T23:56:26,937 INFO [RS:0;b70d29849faf:42393 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T23:56:26,937 INFO [RS:0;b70d29849faf:42393 {}] hbase.ChoreService(370): Chore service for: regionserver/b70d29849faf:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T23:56:26,938 INFO [RS:0;b70d29849faf:42393 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T23:56:26,938 INFO [regionserver/b70d29849faf:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T23:56:26,939 INFO [RS:0;b70d29849faf:42393 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42393 2024-11-11T23:56:26,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b70d29849faf,42393,1731369201430 2024-11-11T23:56:26,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T23:56:26,943 INFO [RS:0;b70d29849faf:42393 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T23:56:26,945 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b70d29849faf,42393,1731369201430] 2024-11-11T23:56:26,947 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b70d29849faf,42393,1731369201430 already deleted, retry=false 2024-11-11T23:56:26,947 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b70d29849faf,42393,1731369201430 expired; onlineServers=0 2024-11-11T23:56:26,947 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b70d29849faf,37327,1731369200577' ***** 2024-11-11T23:56:26,947 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T23:56:26,947 INFO [M:0;b70d29849faf:37327 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T23:56:26,947 INFO [M:0;b70d29849faf:37327 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T23:56:26,948 DEBUG [M:0;b70d29849faf:37327 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T23:56:26,948 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T23:56:26,948 DEBUG [M:0;b70d29849faf:37327 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T23:56:26,948 DEBUG [master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.small.0-1731369203357 {}] cleaner.HFileCleaner(306): Exit Thread[master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.small.0-1731369203357,5,FailOnTimeoutGroup] 2024-11-11T23:56:26,948 DEBUG [master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.large.0-1731369203350 {}] cleaner.HFileCleaner(306): Exit Thread[master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.large.0-1731369203350,5,FailOnTimeoutGroup] 2024-11-11T23:56:26,948 INFO [M:0;b70d29849faf:37327 {}] hbase.ChoreService(370): Chore service for: master/b70d29849faf:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T23:56:26,948 INFO [M:0;b70d29849faf:37327 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T23:56:26,948 DEBUG [M:0;b70d29849faf:37327 {}] master.HMaster(1795): Stopping service threads 2024-11-11T23:56:26,948 INFO [M:0;b70d29849faf:37327 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T23:56:26,949 INFO [M:0;b70d29849faf:37327 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T23:56:26,949 ERROR [M:0;b70d29849faf:37327 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T23:56:26,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T23:56:26,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:56:26,950 INFO [M:0;b70d29849faf:37327 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T23:56:26,950 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T23:56:26,950 DEBUG [M:0;b70d29849faf:37327 {}] zookeeper.ZKUtil(347): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T23:56:26,950 WARN [M:0;b70d29849faf:37327 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T23:56:26,951 INFO [M:0;b70d29849faf:37327 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/.lastflushedseqids 2024-11-11T23:56:26,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742426_1602 (size=72) 2024-11-11T23:56:27,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T23:56:27,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42393-0x1003134c5200001, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T23:56:27,047 INFO [RS:0;b70d29849faf:42393 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T23:56:27,048 INFO [RS:0;b70d29849faf:42393 {}] regionserver.HRegionServer(1031): Exiting; stopping=b70d29849faf,42393,1731369201430; zookeeper connection closed. 2024-11-11T23:56:27,048 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28044495 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28044495 2024-11-11T23:56:27,050 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T23:56:27,361 INFO [M:0;b70d29849faf:37327 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T23:56:27,361 INFO [M:0;b70d29849faf:37327 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T23:56:27,361 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T23:56:27,361 INFO [M:0;b70d29849faf:37327 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:56:27,361 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:56:27,361 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T23:56:27,361 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:56:27,362 INFO [M:0;b70d29849faf:37327 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=748.31 KB heapSize=915.01 KB 2024-11-11T23:56:27,375 DEBUG [M:0;b70d29849faf:37327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/43505eb75f9e4660a492034b9558e55f is 82, key is hbase:meta,,1/info:regioninfo/1731369204376/Put/seqid=0 2024-11-11T23:56:27,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742427_1603 (size=5672) 2024-11-11T23:56:27,779 INFO [M:0;b70d29849faf:37327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2059 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/43505eb75f9e4660a492034b9558e55f 2024-11-11T23:56:27,808 DEBUG [M:0;b70d29849faf:37327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b4f982820dbc4fd9bf3e1120924a86ba is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x1F/proc:d/1731369235517/Put/seqid=0 2024-11-11T23:56:27,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742428_1604 (size=43324) 2024-11-11T23:56:28,213 INFO [M:0;b70d29849faf:37327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=747.75 KB at sequenceid=2059 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b4f982820dbc4fd9bf3e1120924a86ba 2024-11-11T23:56:28,222 INFO [M:0;b70d29849faf:37327 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b4f982820dbc4fd9bf3e1120924a86ba 2024-11-11T23:56:28,244 DEBUG [M:0;b70d29849faf:37327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0df9615315349cfbdc343ec8b880a64 is 69, key is b70d29849faf,42393,1731369201430/rs:state/1731369203430/Put/seqid=0 2024-11-11T23:56:28,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073742429_1605 (size=5156) 2024-11-11T23:56:28,649 INFO [M:0;b70d29849faf:37327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2059 (bloomFilter=true), to=hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0df9615315349cfbdc343ec8b880a64 2024-11-11T23:56:28,659 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/43505eb75f9e4660a492034b9558e55f as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/43505eb75f9e4660a492034b9558e55f 2024-11-11T23:56:28,663 INFO [M:0;b70d29849faf:37327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/43505eb75f9e4660a492034b9558e55f, entries=8, sequenceid=2059, filesize=5.5 K 2024-11-11T23:56:28,663 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b4f982820dbc4fd9bf3e1120924a86ba as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b4f982820dbc4fd9bf3e1120924a86ba 2024-11-11T23:56:28,666 INFO [M:0;b70d29849faf:37327 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b4f982820dbc4fd9bf3e1120924a86ba 2024-11-11T23:56:28,666 INFO [M:0;b70d29849faf:37327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b4f982820dbc4fd9bf3e1120924a86ba, entries=158, sequenceid=2059, filesize=42.3 K 2024-11-11T23:56:28,667 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0df9615315349cfbdc343ec8b880a64 as hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0df9615315349cfbdc343ec8b880a64 2024-11-11T23:56:28,669 INFO [M:0;b70d29849faf:37327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46597/user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0df9615315349cfbdc343ec8b880a64, entries=1, sequenceid=2059, filesize=5.0 K 2024-11-11T23:56:28,670 INFO [M:0;b70d29849faf:37327 {}] regionserver.HRegion(3140): Finished flush of dataSize ~748.31 KB/766268, heapSize ~914.71 KB/936664, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=2059, compaction requested=false 2024-11-11T23:56:28,671 INFO [M:0;b70d29849faf:37327 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:56:28,671 DEBUG [M:0;b70d29849faf:37327 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731369387361Disabling compacts and flushes for region at 1731369387361Disabling writes for close at 1731369387361Obtaining lock to block concurrent updates at 1731369387362 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731369387362Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=766268, getHeapSize=936904, getOffHeapSize=0, getCellsCount=2242 at 1731369387362Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731369387362Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731369387362Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731369387375 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731369387375Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731369387783 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731369387807 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731369387807Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731369388222 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731369388243 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731369388243Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77ed7c86: reopening flushed file at 1731369388658 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68b4ce2c: reopening flushed file at 1731369388663 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@505db160: reopening flushed file at 1731369388666 (+3 ms)Finished flush of dataSize ~748.31 KB/766268, heapSize ~914.71 KB/936664, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=2059, compaction requested=false at 1731369388670 (+4 ms)Writing region close event to WAL at 1731369388671 (+1 ms)Closed at 1731369388671 2024-11-11T23:56:28,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36259 is added to blk_1073741830_1006 (size=901254) 2024-11-11T23:56:28,674 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/43f0e8f7-8fcf-8b3c-05bf-ce1b43270075/MasterData/WALs/b70d29849faf,37327,1731369200577/b70d29849faf%2C37327%2C1731369200577.1731369202805 not finished, retry = 0 2024-11-11T23:56:28,776 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T23:56:28,776 INFO [M:0;b70d29849faf:37327 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T23:56:28,777 INFO [M:0;b70d29849faf:37327 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37327 2024-11-11T23:56:28,777 INFO [M:0;b70d29849faf:37327 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T23:56:28,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T23:56:28,880 INFO [M:0;b70d29849faf:37327 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T23:56:28,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37327-0x1003134c5200000, quorum=127.0.0.1:62745, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T23:56:28,889 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:56:28,894 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T23:56:28,894 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T23:56:28,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T23:56:28,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/hadoop.log.dir/,STOPPED} 2024-11-11T23:56:28,898 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T23:56:28,898 WARN [BP-1069310435-172.17.0.2-1731369198105 heartbeating to localhost/127.0.0.1:46597 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T23:56:28,898 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T23:56:28,898 WARN [BP-1069310435-172.17.0.2-1731369198105 heartbeating to localhost/127.0.0.1:46597 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1069310435-172.17.0.2-1731369198105 (Datanode Uuid 7ec261c1-f9b0-404b-bba9-8b1c60a0fc3a) service to localhost/127.0.0.1:46597 2024-11-11T23:56:28,901 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/cluster_bda0d9c8-fb83-479b-2704-c685aeb4fd07/data/data1/current/BP-1069310435-172.17.0.2-1731369198105 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:56:28,901 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/cluster_bda0d9c8-fb83-479b-2704-c685aeb4fd07/data/data2/current/BP-1069310435-172.17.0.2-1731369198105 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:56:28,902 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T23:56:28,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T23:56:28,908 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T23:56:28,908 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T23:56:28,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T23:56:28,909 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca44b4ef-434d-e437-87d6-f77717eadd5c/hadoop.log.dir/,STOPPED} 2024-11-11T23:56:28,921 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T23:56:29,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down